diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
new file mode 100644
index 000000000000000..8678ff765fbd569
--- /dev/null
+++ b/.azure-pipelines/ci.yml
@@ -0,0 +1,137 @@
+variables:
+ manylinux: false
+ coverage: false
+
+jobs:
+- job: Prebuild
+ displayName: Pre-build checks
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+ displayName: Docs PR
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./docs-steps.yml
+ parameters:
+ upload: true
+
+
+- job: macOS_CI_Tests
+ displayName: macOS CI Tests
+ dependsOn: Prebuild
+ #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+ # bpo-39837: macOS tests on Azure Pipelines are disabled
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-macos'
+ testRunPlatform: macos
+
+ pool:
+ vmImage: macos-10.14
+
+ steps:
+ - template: ./macos-steps.yml
+
+
+- job: Ubuntu_CI_Tests
+ displayName: Ubuntu CI Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-linux'
+ testRunPlatform: linux
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+
+
+- job: ManyLinux1_CI_Tests
+ displayName: ManyLinux1 CI Tests
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['manylinux'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(build.sourceBranchName)-manylinux1'
+ testRunPlatform: manylinux1
+ imageName: 'dockcross/manylinux-x64'
+
+ steps:
+ - template: ./docker-steps.yml
+
+
+- job: Ubuntu_Coverage_CI_Tests
+ displayName: Ubuntu CI Tests (coverage)
+ dependsOn: Prebuild
+ condition: |
+ and(
+ and(
+ succeeded(),
+ eq(variables['coverage'], 'true')
+ ),
+ eq(dependencies.Prebuild.outputs['tests.run'], 'true')
+ )
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
+ testRunPlatform: linux-coverage
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ coverage: true
+
+
+- job: Windows_CI_Tests
+ displayName: Windows CI Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: vs2017-win2016
+
+ strategy:
+ matrix:
+ win32:
+ arch: win32
+ buildOpt:
+ testRunTitle: '$(Build.SourceBranchName)-win32'
+ testRunPlatform: win32
+ win64:
+ arch: amd64
+ buildOpt: '-p x64'
+ testRunTitle: '$(Build.SourceBranchName)-win64'
+ testRunPlatform: win64
+ maxParallel: 2
+
+ steps:
+ - template: ./windows-steps.yml
diff --git a/.azure-pipelines/docker-steps.yml b/.azure-pipelines/docker-steps.yml
new file mode 100644
index 000000000000000..ba4dfd72dd8bb8b
--- /dev/null
+++ b/.azure-pipelines/docker-steps.yml
@@ -0,0 +1,76 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- ${{ if ne(parameters.targetBranch, '') }}:
+ - script: |
+ git fetch -q origin ${{ parameters.targetbranch }}
+ if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ echo "##vso[task.setvariable variable=DocOnly]true"
+ exit
+ fi
+ displayName: Detect doc-only changes
+
+- task: docker@0
+ displayName: 'Configure CPython (debug)'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: './configure --with-pydebug'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Build CPython'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make -s -j4'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Display build info'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make pythoninfo'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: docker@0
+ displayName: 'Tests'
+ inputs:
+ action: 'Run an image'
+ imageName: $(imageName)
+ volumes: |
+ $(build.sourcesDirectory):/src
+ $(build.binariesDirectory):/build
+ workDir: '/src'
+ containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"'
+ detached: false
+ condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))
diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml
new file mode 100644
index 000000000000000..492e4e34bb2dab9
--- /dev/null
+++ b/.azure-pipelines/docs-steps.yml
@@ -0,0 +1,46 @@
+parameters:
+ latex: false
+ upload: false
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- task: UsePythonVersion@0
+ displayName: 'Use Python 3.6 or later'
+ inputs:
+ versionSpec: '>=3.6'
+
+- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme
+ displayName: 'Install build dependencies'
+
+- ${{ if ne(parameters.latex, 'true') }}:
+ - script: make check suspicious html PYTHON=python
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+
+- ${{ if eq(parameters.latex, 'true') }}:
+ - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full
+ displayName: 'Install LaTeX'
+
+ - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb'
+ workingDirectory: '$(build.sourcesDirectory)/Doc'
+ displayName: 'Build documentation'
+
+- ${{ if eq(parameters.upload, 'true') }}:
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish docs'
+
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/build'
+ ArtifactName: docs
+ publishLocation: Container
+
+ - ${{ if eq(parameters.latex, 'true') }}:
+ - task: PublishBuildArtifacts@1
+ displayName: 'Publish dist'
+ inputs:
+ PathToPublish: '$(build.sourcesDirectory)/Doc/dist'
+ ArtifactName: docs_dist
+ publishLocation: Container
diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml
new file mode 100644
index 000000000000000..fa38a0df8c87b82
--- /dev/null
+++ b/.azure-pipelines/macos-steps.yml
@@ -0,0 +1,27 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
+ displayName: 'Configure CPython (debug)'
+
+- script: make -j4
+ displayName: 'Build CPython'
+
+- script: make pythoninfo
+ displayName: 'Display build info'
+
+- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+ displayName: 'Tests'
+ continueOnError: true
+ timeoutInMinutes: 30
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/posix-deps.sh b/.azure-pipelines/posix-deps.sh
new file mode 100755
index 000000000000000..a572107566016ec
--- /dev/null
+++ b/.azure-pipelines/posix-deps.sh
@@ -0,0 +1,26 @@
+sudo apt-get update
+
+sudo apt-get -yq install \
+ build-essential \
+ zlib1g-dev \
+ libbz2-dev \
+ liblzma-dev \
+ libncurses5-dev \
+ libreadline6-dev \
+ libsqlite3-dev \
+ libssl-dev \
+ libgdbm-dev \
+ tk-dev \
+ lzma \
+ lzma-dev \
+ liblzma-dev \
+ libffi-dev \
+ uuid-dev \
+ xvfb
+
+if [ ! -z "$1" ]
+then
+ echo ##vso[task.prependpath]$PWD/multissl/openssl/$1
+ echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1
+ python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux
+fi
diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml
new file mode 100644
index 000000000000000..9fec9be8014ffd6
--- /dev/null
+++ b/.azure-pipelines/posix-steps.yml
@@ -0,0 +1,63 @@
+parameters:
+ coverage: false
+
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- script: ./.azure-pipelines/posix-deps.sh $(openssl_version)
+ displayName: 'Install dependencies'
+
+- script: ./configure --with-pydebug
+ displayName: 'Configure CPython (debug)'
+
+- script: make -s -j4
+ displayName: 'Build CPython'
+
+- ${{ if eq(parameters.coverage, 'true') }}:
+ - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage
+ displayName: 'Set up virtual environment'
+
+ - script: ./venv/bin/python -m test.pythoninfo
+ displayName: 'Display build info'
+
+ - script: |
+ xvfb-run ./venv/bin/python -m coverage run --pylib -m test \
+ --fail-env-changed \
+ -uall,-cpu \
+ --junit-xml=$(build.binariesDirectory)/test-results.xml" \
+ -x test_multiprocessing_fork \
+ -x test_multiprocessing_forkserver \
+ -x test_multiprocessing_spawn \
+ -x test_concurrent_futures
+ displayName: 'Tests with coverage'
+
+ - script: ./venv/bin/python -m coverage xml
+ displayName: 'Generate coverage.xml'
+
+ - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash)
+ displayName: 'Publish code coverage results'
+
+
+- ${{ if ne(parameters.coverage, 'true') }}:
+ - script: make pythoninfo
+ displayName: 'Display build info'
+
+ - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
+ displayName: 'Tests'
+
+
+- script: ./python Tools/scripts/patchcheck.py --travis true
+ displayName: 'Run patchcheck.py'
+ condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
+
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml
new file mode 100644
index 000000000000000..6db2a0459200217
--- /dev/null
+++ b/.azure-pipelines/pr.yml
@@ -0,0 +1,88 @@
+jobs:
+- job: Prebuild
+ displayName: Pre-build checks
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./prebuild-checks.yml
+
+
+- job: Docs_PR
+ displayName: Docs PR
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ steps:
+ - template: ./docs-steps.yml
+
+
+- job: macOS_PR_Tests
+ displayName: macOS PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+ #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+ # bpo-39837: macOS tests on Azure Pipelines are disabled
+
+ variables:
+ testRunTitle: '$(system.pullRequest.TargetBranch)-macos'
+ testRunPlatform: macos
+
+ pool:
+ vmImage: macos-10.14
+
+ steps:
+ - template: ./macos-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Ubuntu_PR_Tests
+ displayName: Ubuntu PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: ubuntu-16.04
+
+ variables:
+ testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
+ testRunPlatform: linux
+ openssl_version: 1.1.0g
+
+ steps:
+ - template: ./posix-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
+
+
+- job: Windows_PR_Tests
+ displayName: Windows PR Tests
+ dependsOn: Prebuild
+ condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+
+ pool:
+ vmImage: vs2017-win2016
+
+ strategy:
+ matrix:
+ win32:
+ arch: win32
+ buildOpt:
+ testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
+ testRunPlatform: win32
+ win64:
+ arch: amd64
+ buildOpt: '-p x64'
+ testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
+ testRunPlatform: win64
+ maxParallel: 2
+
+ steps:
+ - template: ./windows-steps.yml
+ parameters:
+ targetBranch: $(System.PullRequest.TargetBranch)
diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml
new file mode 100644
index 000000000000000..30ff642d1267a1e
--- /dev/null
+++ b/.azure-pipelines/prebuild-checks.yml
@@ -0,0 +1,36 @@
+steps:
+- checkout: self
+ fetchDepth: 5
+
+- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1"
+ displayName: Set default diff target
+
+- script: |
+ git fetch -q origin $(System.PullRequest.TargetBranch)
+ echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)"
+ displayName: Fetch comparison tree
+ condition: and(succeeded(), variables['System.PullRequest.TargetBranch'])
+
+- script: |
+ if ! git diff --name-only $(diffTarget) | grep -qE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "No docs were updated: docs.run=false"
+ echo "##vso[task.setvariable variable=run;isOutput=true]false"
+ else
+ echo "Docs were updated: docs.run=true"
+ echo "##vso[task.setvariable variable=run;isOutput=true]true"
+ fi
+ displayName: Detect documentation changes
+ name: docs
+
+- script: |
+ if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)'
+ then
+ echo "Only docs were updated: tests.run=false"
+ echo "##vso[task.setvariable variable=run;isOutput=true]false"
+ else
+ echo "Code was updated: tests.run=true"
+ echo "##vso[task.setvariable variable=run;isOutput=true]true"
+ fi
+ displayName: Detect source changes
+ name: tests
diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml
new file mode 100644
index 000000000000000..d8d5f1753a07172
--- /dev/null
+++ b/.azure-pipelines/windows-steps.yml
@@ -0,0 +1,32 @@
+steps:
+- checkout: self
+ clean: true
+ fetchDepth: 5
+
+- powershell: |
+ # Relocate build outputs outside of source directory to make cleaning faster
+ Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj'
+ # UNDONE: Do not build to a different directory because of broken tests
+ Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild'
+ Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals'
+ displayName: Update build locations
+
+- script: PCbuild\build.bat -e $(buildOpt)
+ displayName: 'Build CPython'
+
+- script: python.bat -m test.pythoninfo
+ displayName: 'Display build info'
+
+- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml"
+ displayName: 'Tests'
+ env:
+ PREFIX: $(Py_OutDir)\$(arch)
+
+- task: PublishTestResults@2
+ displayName: 'Publish Test Results'
+ inputs:
+ testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
+ mergeTestResults: true
+ testRunTitle: $(testRunTitle)
+ platform: $(testRunPlatform)
+ condition: succeededOrFailed()
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 000000000000000..e3dbd9d3a2fab73
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,58 @@
+# Binary data types
+*.aif binary
+*.aifc binary
+*.aiff binary
+*.au binary
+*.bmp binary
+*.exe binary
+*.icns binary
+*.gif binary
+*.ico binary
+*.jpg binary
+*.pck binary
+*.png binary
+*.psd binary
+*.tar binary
+*.wav binary
+*.whl binary
+*.zip binary
+
+# Specific binary files
+Lib/test/sndhdrdata/sndhdr.* binary
+
+# Text files that should not be subject to eol conversion
+Lib/test/cjkencodings/* -text
+Lib/test/decimaltestdata/*.decTest -text
+Lib/test/test_email/data/*.txt -text
+Lib/test/xmltestdata/* -text
+Lib/test/coding20731.py -text
+
+# Special files in third party code
+Modules/zlib/zlib.map -text
+
+# CRLF files
+*.bat text eol=crlf
+*.ps1 text eol=crlf
+*.sln text eol=crlf
+*.vcxproj* text eol=crlf
+*.props text eol=crlf
+*.proj text eol=crlf
+PCbuild/readme.txt text eol=crlf
+PC/readme.txt text eol=crlf
+
+# Generated files
+# https://github.com/github/linguist#generated-code
+Include/graminit.h linguist-generated=true
+Python/graminit.h linguist-generated=true
+Modules/clinic/*.h linguist-generated=true
+Objects/clinic/*.h linguist-generated=true
+PC/clinic/*.h linguist-generated=true
+Python/clinic/*.h linguist-generated=true
+Python/importlib.h linguist-generated=true
+Python/importlib_external.h linguist-generated=true
+Include/Python-ast.h linguist-generated=true
+Python/Python-ast.c linguist-generated=true
+Include/opcode.h linguist-generated=true
+Python/opcode_targets.h linguist-generated=true
+Objects/typeslots.inc linguist-generated=true
+Modules/unicodedata_db.h linguist-generated=true
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000000000..84c534145de8ea1
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,64 @@
+# See https://help.github.com/articles/about-codeowners/
+# for more info about CODEOWNERS file
+
+# It uses the same pattern rule for gitignore file
+# https://git-scm.com/docs/gitignore#_pattern_format
+
+# asyncio
+**/*asyncio* @1st1 @asvetlov
+
+# Core
+**/*context* @1st1
+**/*genobject* @1st1
+**/*hamt* @1st1
+
+# Hashing
+**/*hashlib* @python/crypto-team
+**/*pyhash* @python/crypto-team
+
+# Import (including importlib).
+# Ignoring importlib.h so as to not get flagged on
+# all pull requests that change the emitted
+# bytecode.
+**/*import*.c @python/import-team
+**/*import*.py @python/import-team
+
+
+# SSL
+**/*ssl* @python/crypto-team
+
+# CSPRNG
+Python/bootstrap_hash.c @python/crypto-team
+
+# Email and related
+**/*mail* @python/email-team
+**/*smtp* @python/email-team
+**/*mime* @python/email-team
+**/*imap* @python/email-team
+**/*poplib* @python/email-team
+
+# subprocess
+**/*subprocess* @gpshead
+
+# Windows
+/PC/ @python/windows-team
+/PCbuild/ @python/windows-team
+
+# Windows installer packages
+/Tools/msi/ @python/windows-team
+/Tools/nuget/ @python/windows-team
+
+**/*itertools* @rhettinger
+**/*collections* @rhettinger
+**/*random* @rhettinger
+**/*queue* @rhettinger
+**/*bisect* @rhettinger
+**/*heapq* @rhettinger
+**/*functools* @ncoghlan @rhettinger
+**/*decimal* @rhettinger @skrah
+
+**/*dataclasses* @ericvsmith
+
+**/*idlelib* @terryjreedy
+
+**/*typing* @gvanrossum @ilevkivskyi
diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst
new file mode 100644
index 000000000000000..7f912e870841902
--- /dev/null
+++ b/.github/CONTRIBUTING.rst
@@ -0,0 +1,67 @@
+Contributing to Python
+======================
+
+Build Status
+------------
+
+- master
+
+ + `Stable buildbots `_
+
+- 3.7
+
+ + `Stable buildbots `_
+
+- 3.6
+
+ + `Stable buildbots `_
+
+- 2.7
+
+ + `Stable buildbots `_
+
+
+Thank You
+---------
+First off, thanks for contributing to the maintenance of the Python programming
+language and the CPython interpreter! Even if your contribution is not
+ultimately accepted, the fact you put time and effort into helping out is
+greatly appreciated.
+
+
+Contribution Guidelines
+-----------------------
+Please read the `devguide `_ for
+guidance on how to contribute to this project. The documentation covers
+everything from how to build the code to submitting a pull request. There are
+also suggestions on how you can most effectively help the project.
+
+Please be aware that our workflow does deviate slightly from the typical GitHub
+project. Details on how to properly submit a pull request are covered in
+`Lifecycle of a Pull Request `_.
+We utilize various bots and status checks to help with this, so do follow the
+comments they leave and their "Details" links, respectively. The key points of
+our workflow that are not covered by a bot or status check are:
+
+- All discussions that are not directly related to the code in the pull request
+ should happen on bugs.python.org
+- Upon your first non-trivial pull request (which includes documentation changes),
+ feel free to add yourself to ``Misc/ACKS``
+
+
+Setting Expectations
+--------------------
+Due to the fact that this project is entirely volunteer-run (i.e. no one is paid
+to work on Python full-time), we unfortunately can make no guarantees as to if
+or when a core developer will get around to reviewing your pull request.
+If no core developer has done a review or responded to changes made because of a
+"changes requested" review, please feel free to email python-dev to ask if
+someone could take a look at your pull request.
+
+
+Code of Conduct
+---------------
+All interactions for this project are covered by the
+`PSF Code of Conduct `_. Everyone is
+expected to be open, considerate, and respectful of others no matter their
+position within the project.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 4ce80d872df7b6f..55e4168747e10d1 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,9 +1,12 @@
-## CPython Mirror
+!!! If this is a backport PR (PR made against branches other than `master`),
+please ensure that the PR title is in the following format:
-https://github.com/python/cpython is a cpython mirror repository. Pull requests
-are not accepted on this repo and will be automatically closed.
+```
+[X.Y]
(GH-NNNN)
+```
-### Submit patches at https://bugs.python.org
+Where: [X.Y] is the branch name, e.g. [3.7].
-For additional information about contributing to CPython, see the
-[developer's guide](https://docs.python.org/devguide/#contributing).
+GH-NNNN refers to the PR number from `master`.
+
+PLEASE: Remove this headline!!!
diff --git a/.github/posix-deps-zypp.sh b/.github/posix-deps-zypp.sh
new file mode 100755
index 000000000000000..fb2a9377c84c727
--- /dev/null
+++ b/.github/posix-deps-zypp.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+set -eu
+
+zypper --non-interactive refresh
+
+# The container can have newer libraries than the repositories' matching -devel
+# packages. Allow the solver to downgrade dependencies rather than cancel.
+# Install build tools explicitly: build patterns pull in a full base system,
+# including packages that conflict with the container's busybox replacements.
+# Python 3.6 needs OpenSSL 1.1 headers rather than Leap's default OpenSSL 3.
+# GDB needs the UTF-32 converters supplied by glibc-locale-base.
+zypper --non-interactive install --auto-agree-with-licenses \
+ --allow-downgrade --no-recommends \
+ autoconf automake gcc gcc-c++ make patch pkg-config python3 \
+ binutils diffutils findutils gdb glibc-locale-base gzip libtool perl tar which \
+ libabigail-tools xorg-x11-server-Xvfb xvfb-run \
+ cantarell-fonts google-droid-fonts google-inconsolata-fonts dejavu-fonts \
+ libexpat-devel \
+ libffi-devel \
+ xz-devel \
+ bzip2 \
+ zlib-devel \
+ libbz2-devel \
+ ncurses-devel \
+ readline-devel \
+ sqlite3-devel \
+ libopenssl-1_1-devel \
+ gdbm-devel \
+ tk-devel \
+ libuuid-devel \
+ lcov
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 000000000000000..13a1a26e74204b4
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,129 @@
+name: Tests
+
+# Cancel in-progress runs for the same PR.
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+on:
+ pull_request:
+
+jobs:
+ check_source:
+ name: 'Check for source changes'
+ runs-on: ubuntu-latest
+ outputs:
+ run_tests: ${{ steps.check.outputs.run_tests }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 1000
+ - name: Check for source changes
+ id: check
+ run: |
+ if [ -z "$GITHUB_BASE_REF" ]; then
+ echo "run_tests=true" >> "$GITHUB_OUTPUT"
+ else
+ git fetch origin $GITHUB_BASE_REF --depth=1
+ git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> "$GITHUB_OUTPUT" || true
+ fi
+
+ opensuse-tests:
+ name: '${{ matrix.name }} (openSUSE Leap)'
+ runs-on: ubuntu-latest
+ needs: check_source
+ if: needs.check_source.outputs.run_tests == 'true'
+ # Use a specific image version for reproducible builds
+ container:
+ image: opensuse/leap:15.6
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: 'Check ABI'
+ task: 'abi'
+ - name: 'Check Generated Files'
+ task: 'generated-files'
+ - name: 'Build and Test'
+ task: 'build-and-test'
+
+ env:
+ OPENSSL_VER: 1.1.1u
+
+ steps:
+
+ - name: Install Git and Tar
+ run: |
+ zypper ref
+ zypper -n install -y git-core tar
+
+ - uses: actions/checkout@v4
+
+ - name: Trust the checked-out workspace
+ # checkout configures a temporary HOME, not the HOME used by shell steps.
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+ - name: Install All Dependencies
+ run: sh -x ./.github/posix-deps-zypp.sh
+
+ - name: 'Restore OpenSSL build'
+ if: matrix.task == 'build-and-test'
+ id: cache-openssl
+ uses: actions/cache@v4
+ with:
+ path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+ key: opensuse-leap-multissl-openssl-${{ env.OPENSSL_VER }}
+
+ # === Steps for ABI Check ===
+ - name: Build CPython for ABI Check
+ if: matrix.task == 'abi'
+ env:
+ CFLAGS: -g3 -O0
+ run: |
+ ./configure --enable-shared --with-fpectl --with-system-expat
+ make -j4
+
+ - name: Check for changes in the ABI
+ if: matrix.task == 'abi'
+ run: make check-abidump
+
+ # === Steps for Generated Files Check ===
+ - name: Build CPython for Generated Files Check
+ if: matrix.task == 'generated-files'
+ run: |
+ ./configure --with-pydebug --with-system-expat
+ make -j4 regen-all
+
+ - name: Check for changes in generated files
+ if: matrix.task == 'generated-files'
+ run: |
+ git status --short
+ git diff --exit-code
+
+ - name: Check exported libpython symbols
+ if: matrix.task == 'generated-files'
+ run: make smelly
+
+ # === Steps for Full Build and Test ===
+ - name: Install OpenSSL
+ if: matrix.task == 'build-and-test' && steps.cache-openssl.outputs.cache-hit != 'true'
+ run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux
+
+ - name: Configure CPython
+ if: matrix.task == 'build-and-test'
+ run: |
+ autoconf -Wall
+ autoheader -Wall
+ ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER --with-system-expat
+
+ - name: Build CPython
+ if: matrix.task == 'build-and-test'
+ run: make -j4
+
+ - name: Display build info
+ if: matrix.task == 'build-and-test'
+ run: make pythoninfo
+
+ - name: Run tests
+ if: matrix.task == 'build-and-test'
+ run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
new file mode 100644
index 000000000000000..1c5ac25f53413a7
--- /dev/null
+++ b/.github/workflows/doc.yml
@@ -0,0 +1,50 @@
+name: Docs
+
+on:
+ #push:
+ # branches:
+ # - master
+ # - 3.8
+ # - 3.7
+ # paths:
+ # - 'Doc/**'
+ pull_request:
+ paths:
+ - 'Doc/**'
+ - 'Misc/**'
+
+jobs:
+ build_doc:
+ name: 'Docs'
+ runs-on: ubuntu-latest
+ container:
+ image: opensuse/leap:15.6
+ steps:
+ - name: Install Git and Tar
+ run: |
+ zypper ref
+ zypper -n install -y git-core tar
+
+ - uses: actions/checkout@v4
+
+ - name: 'Install Dependencies'
+ run: sh ./.github/posix-deps-zypp.sh
+ # apt-get install wamerican
+
+ - name: 'Configure CPython'
+ run: ./configure --with-pydebug --with-system-expat
+
+ - name: 'Build CPython'
+ run: make -j4
+
+ - name: 'Install build dependencies'
+ run: make -C Doc/ PYTHON=../python venv
+
+ - name: 'Build documentation'
+ run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" suspicious html
+
+ - name: 'Upload'
+ uses: actions/upload-artifact@v4
+ with:
+ name: doc-html
+ path: Doc/build/html
diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml
new file mode 100644
index 000000000000000..773eba5d245b85e
--- /dev/null
+++ b/.github/workflows/verify-ensurepip-wheels.yml
@@ -0,0 +1,28 @@
+name: Verify bundled pip and setuptools
+
+on:
+ workflow_dispatch:
+ push:
+ paths:
+ - 'Lib/ensurepip/_bundled/**'
+ - '.github/workflows/verify-ensurepip-wheels.yml'
+ - 'Tools/scripts/verify_ensurepip_wheels.py'
+ pull_request:
+ paths:
+ - 'Lib/ensurepip/_bundled/**'
+ - '.github/workflows/verify-ensurepip-wheels.yml'
+ - 'Tools/scripts/verify_ensurepip_wheels.py'
+
+permissions:
+ contents: read
+
+jobs:
+ verify:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3'
+ - name: Compare checksums of bundled pip and setuptools to ones published on PyPI
+ run: python3 ./Tools/scripts/verify_ensurepip_wheels.py
diff --git a/.gitignore b/.gitignore
index ed4ebfbbd9de74e..945ee76c609af2f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,10 @@
.gdb_history
Doc/build/
Doc/venv/
+Doc/.venv/
+Doc/env/
+Doc/.env/
+Include/pydtrace_probes.h
Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
@@ -51,10 +55,12 @@ PCbuild/*.suo
PCbuild/*.*sdf
PCbuild/*-pgi
PCbuild/*-pgo
+PCbuild/*.VC.db
+PCbuild/*.VC.opendb
PCbuild/.vs/
PCbuild/amd64/
PCbuild/obj/
-PCBuild/win32/
+PCbuild/win32/
.purify
Parser/pgen
__pycache__
@@ -94,3 +100,4 @@ Tools/ssl/amd64
Tools/ssl/win32
.vs/
.vscode/
+gmon.out
diff --git a/.hgeol b/.hgeol
deleted file mode 100644
index eb19a6c88d28d05..000000000000000
--- a/.hgeol
+++ /dev/null
@@ -1,58 +0,0 @@
-[patterns]
-
-# Non human-editable files are binary
-
-**.dsp = BIN
-**.dsw = BIN
-**.mk = BIN
-**.sln = BIN
-**.vcproj = BIN
-**.vsprops = BIN
-
-**.aif = BIN
-**.aifc = BIN
-**.aiff = BIN
-**.au = BIN
-**.bmp = BIN
-**.db = BIN
-**.exe = BIN
-**.icns = BIN
-**.gif = BIN
-**.ico = BIN
-**.info = BIN
-**.jpg = BIN
-**.pck = BIN
-**.png = BIN
-**.psd = BIN
-**.tar = BIN
-**.wav = BIN
-**.whl = BIN
-**.xar = BIN
-**.zip = BIN
-
-Lib/test/cjkencodings/* = BIN
-Lib/test/decimaltestdata/*.decTest = BIN
-Lib/test/sndhdrdata/sndhdr.* = BIN
-Lib/test/test_email/data/msg_26.txt = BIN
-Lib/test/xmltestdata/* = BIN
-
-Lib/venv/scripts/nt/* = BIN
-
-Lib/test/coding20731.py = BIN
-
-# Windows only zlib upstream file
-Modules/zlib/zlib.map = CRLF
-
-# Windows batch files work best with CRLF, there can be subtle problems with LF
-**.bat = CRLF
-
-# The Windows readme is likely to be read in Notepad, so make it readable
-PCbuild/readme.txt = CRLF
-
-# All other files (which presumably are human-editable) are "native".
-# This must be the last rule!
-
-** = native
-
-[repository]
-native = LF
diff --git a/.hgtags b/.hgtags
deleted file mode 100644
index 8f51c2ced49aed4..000000000000000
--- a/.hgtags
+++ /dev/null
@@ -1,182 +0,0 @@
-64cc5439e10a6fdf984effaf0141e94fa4cc1004 v0.9.8
-78a7ed6953025e7ecdde9585099b01a6ae40b76a v0.9.9
-b15b8cc9b8d10e0352a0b8b7e8d51fa309db6df3 v1.0.1
-0326b5d61445ee3a8d3de28119f9652cb72d2e3f v1.0.2
-832615ec07646e310c85316b8ba6bc9b17ad3547 v1.1
-9895475d18c7b5f32adaf78f71886ae041e4d10c v1.1.1
-16eb4c51ee97169046340998e850a63c65225b0a v1.2b1
-b45c688756d04fb84d4a0d518fc3d7e3cb25fa8d v1.2b2
-9e82daf7605bad7976a9abc997cb5e0abe434078 v1.2b3
-065e31cf5862e27521cf5725b003aed211f091b2 v1.2b4
-e72257e655454d569468da8b1189e0ec336f3536 v1.2
-e63d83f8275853aaaa3d1972cb86564505e65583 v1.3b1
-7d743c865a9aa6bde8b603e32e0542031bba3c33 v1.3
-4fc85c82cc222554ae6b9c0b87776ed5f2b70c6e v1.4b1
-129f1299d4e97e884bbbbdd00baf101d178973e6 v1.4b2
-44a82ac654a4175569deed8e8a94b0cc8edee08d v1.4b3
-db49494c93dc73de06d5721c74eab533a947a92c v1.4
-062aed8a4ce2c91c81b80e29f02faff1cf5a761b v1.5a1
-c9498ac988372575cf7028b86395b900c9b0a840 v1.5a2
-dc5c968ec992aab3d40a7189df0c443d1c7a1a68 v1.5a3
-746654a0af680c7d9b814b210a026eb91bec9533 v1.5a4
-8ff58b5730f06be08fbbdc2bf592226f7a736201 v1.5b1
-eb78658d819fb0af09a8e6f9bedcb670805ed5f6 v1.5b2
-84461011a1a0ab402e352f06748f29fb5b5559e5 v1.5
-44aba4d26b01fbae0403efe654f9fd0347606732 v1.5.1
-fed63ccbe6dc3ac663bfe97a2f7006b1b28568f9 v1.5.2a1
-21d71f2e27248a0f4e393d0fc321ecf9b89321d2 v1.5.2a2
-f08c7a2a56f80741f5f192fd0ebe0b0967a203cf v1.5.2b1
-8fe7ec4b4fc1518fcac89e6bf674fbbce16150a9 v1.5.2b2
-39fb0dcc83dc375c1565ba65dbce0ed59b1359c9 v1.5.2c1
-61c91c7f101bab3149adfcd5646ae40e048de712 v1.5.2
-605eb9326ffe1fd1e43f40e2338d6652ab449fdf v1.6a1
-011bee8fd9f7f4da457ec71596484fb0882c0614 v1.6a2
-35c4fc1414a59888614b9be784a25f233ba67984 v2.0b1
-55bba197d4870cdae62aeca00e20240a756b84f8 v2.0b2
-e276329cce036a5f9e9d3451256dca5984e543dc v2.0c1
-2fa4e35083e02342ca014bf5bfba46aecb816c31 v2.0
-b60831eeab5a06dd3c5e8297a99e39297aa8794b v2.1a1
-b382f1f07ec6b2c95551658b30c6139eeb32077a v2.1a2
-d0c830db5e68edd4aaa3401216e610c9ff145826 v2.1b1
-b59a536ae1ef3774fd85c17f623e8926b7b6c095 v2.1b2
-d611276e9ad53b5d32d1e8065e1d811c32f7d96f v2.1c1
-ff065e674af6c9ab895bd9eff7d9e9039a376c7d v2.1c2
-020e95d8180d7943fe54701e1db0a7d7d87e2b1e v2.1
-08796a137f1ada2462f7a3177306df5f67a767e1 v2.2a3
-d054c29647f90bccb8345bd779bca1eecf2dd7f2 v2.3c1
-fce5c9e9abc722394cb2e909b7e2a39080d4448e v2.3c2
-92ca658fd420095b6284c9ce6e9082a80285ec9c v2.4a1
-055fc6955f3c6522bfeb7ed4c671c97d5baaaac2 v2.4a2
-186b72550e53533ef6175f6411f932c1298193d7 v2.4a3
-53cff04283855adf88ed0c0fd3698827ca843637 v2.4b1
-7e387a9dcc79954a77695adef8b593da35be1214 v2.4b2
-ff80d8bbef6e13426c8a85d7f9d837b8f8f89834 v2.4c1
-f31e18d313c7a4fc66914b2d27e130a0f72c0b69 v2.4
-cd3f783cd08a16781e236c0b9cb5717d1d995fa9 v3.0a1
-65e82140e281bf26f2e22eda05a7f9956c420f8b v3.0a2
-df15827f34881b9af0936350813ced5c123c8230 v3.0a3
-15f773f7300e372c56a21d59fe49ca26955a6477 v3.0a4
-e35935475153377d6727d64e6c52f72c3b84015b v3.0a5
-a335c4d643b1cfe14197a9ef195c9b2804f608fc v3.0b1
-16ec4bb14a68ea428acf09ebf0c92981da2646f3 v3.0b2
-509e30a7968e01be329ec121540b3e755fc4e566 v3.0b3
-507ede9c7f7f475dfafbd4a52c22d767d10a2bc0 v3.0rc1
-8fae465a39627b590385462e6095eb63af45240a v3.0rc2
-e83a60c69d53f5551a306e77a6d38e9b11485496 v3.0rc3
-bc1ce368986e45b1faf96f93995df46bcd75e7b8 v3.1a1
-ee430e5075db2adf8124e6b94916a89ca41d3171 v3.1a2
-b63020797f9678adaf4d2c3e26574a9eef2ef028 v3.1b1
-4353fd0843cb31b356adc50f93d220e2e7255ef2 v3.1rc1
-0b87e438e1b53e3f812cad963a7fdb65d198ba2f v3.1rc2
-a69a031ac1402dede8b1ef80096436bca6d371f3 v3.1
-35efb1054ec6ceca72017a587263cb6a9257340b v3.1.1rc1
-8b9c0f573ab29c41c6c5fdcca82a1fe0ff5355af v3.1.1
-149b8b87514d10416b598884db5f74651f625b38 v3.1.2rc1
-960efa327c5d9c18df995437b0ac550cb89c9f85 v3.1.2
-d18e9d71f369d8211f6ac87252c6d3211f9bd09f v3.1.3rc1
-a4f75773c0060cee38b0bb651a7aba6f56b0e996 v3.1.3
-32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1
-c918ec9f3a76d6afedfbb5d455004de880443a3d v3.1.4
-ee26aca3219cf4bb0b93352e83edcc9cb28c7802 v3.1.5rc1
-75db2bc69fc9a3e4801e94e3e19801cb096208d8 v3.1.5rc2
-7395330e495ec3316862ca1f6ce0aaf7bdf6785b v3.1.5
-b37b7834757492d009b99cf0ca4d42d2153d7fac v3.2a1
-56d4373cecb73c8b45126ba7b045b3c7b3f94b0b v3.2a2
-da012d9a2c23d144e399d2e01a55b8a83ad94573 v3.2a3
-d92a5b850f5e56808bedc01723906ed64c5e6e2e v3.2a4
-b635cea94195780c8716e236479af319bcc26253 v3.2b1
-e3af5f3a7904c0d5343ec9633ea66e7acfd23a66 v3.2b2
-865d5b24bf28ca41b536befc326407c03e74a4d5 v3.2rc1
-acf3e24dd0d0dfd1e20c907d696d3da965a8f56f v3.2rc2
-18c1f52896501c7ee13b038454a39acb45a87979 v3.2rc3
-a222a015e28d8ae9af3899258dc6c15c3d40add0 v3.2
-8ffac2337a3323323d02153ac919fd1483176652 v3.2.1b1
-cfa9364997c7f2e67b9cbb45c3a5fa3bba4e4999 v3.2.1rc1
-5df549718fb4841ff521fe051f6b54f290fad5d8 v3.2.1rc2
-ac1f7e5c05104d557d5acd922e95625ba5d1fe10 v3.2.1
-c860feaa348d663e598986894ee4680480577e15 v3.2.2rc1
-137e45f15c0bd262c9ad4c032d97425bc0589456 v3.2.2
-7085403daf439adb3f9e70ef13f6bedb1c447376 v3.2.3rc1
-428f05cb7277e1d42bb9dd8d1af6b6270ebc6112 v3.2.3rc2
-3d0686d90f55a78f96d9403da2c52dc2411419d0 v3.2.3
-b2cb7bc1edb8493c0a78f9331eae3e8fba6a881d v3.2.4rc1
-1e10bdeabe3de02f038a63c001911561ac1d13a7 v3.2.4
-cef745775b6583446572cffad704100983db2bea v3.2.5
-51382a5598ec96119cb84594572901c9c964dc3c v3.2.6rc1
-0bd5f4f14de965ca8e44c6e3965fee106176cfc4 v3.2.6
-f1a9a6505731714f0e157453ff850e3b71615c45 v3.3.0a1
-2f69db52d6de306cdaef0a0cc00cc823fb350b01 v3.3.0a2
-0b53b70a40a00013505eb35e3660057b62be77be v3.3.0a3
-7c51388a3aa7ce76a8541bbbdfc05d2d259a162c v3.3.0a4
-e15c554cd43eb23bc0a528a4e8741da9bbec9607 v3.3.0b1
-4972a8f1b2aa3d7cdd64dc96aa7fa112fe1ea343 v3.3.0b2
-8bb5c7bc46ba43804480f3e328e1fa956672c885 v3.3.0rc1
-88a0792e8ba3e4916b24c7e7a522c277d326d66e v3.3.0rc2
-c191d21cefafb3832c45570e84854e309aa62eaa v3.3.0rc3
-bd8afb90ebf28ba4edc901d4a235f75e7bbc79fd v3.3.0
-92c2cfb924055ce68c4f78f836dcfe688437ceb8 v3.3.1rc1
-d9893d13c6289aa03d33559ec67f97dcbf5c9e3c v3.3.1
-d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2
-fd53c500f8b80f54f3ecedec9da2e8c7e52a6888 v3.3.3rc1
-d32442c0e60dfbd71234e807d3d1dedd227495a9 v3.3.3rc2
-c3896275c0f61b2510a6c7e6c458a750359a91b8 v3.3.3
-fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1
-7ff62415e4263c432c8acf6e424224209211eadb v3.3.4
-9ec811df548ed154a9bf9815383a916d6df31b98 v3.3.5rc1
-ca5635efe090f78806188ac2758f9948596aa8b2 v3.3.5rc2
-62cf4e77f78564714e7ea3d4bf1479ca1fbd0758 v3.3.5
-51317c9786f54267975abf2e9c502e6aaaa4a249 v3.3.6rc1
-971fec30da1fc5bf2b9fb28e09812a5127014211 v3.3.6
-46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1
-9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2
-dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3
-e245b0d7209bb6d0e19316e1e2af1aa9c2139104 v3.4.0a4
-3405dc9a6afaa0a06dd1f6f182ec5c998dce6f5f v3.4.0b1
-ba32913eb13ec545a46dd0ce18035b6c416f0d78 v3.4.0b2
-a97ce3ecc96af79bd2e1ac66ce48d9138e0ca749 v3.4.0b3
-5e088cea8660677969113741c1313d570d977e02 v3.4.0rc1
-a300712ed38c9a242b736c44e806caea25a6dc05 v3.4.0rc2
-8a81cdab3e9d521daaef989fade94b16455fc3b8 v3.4.0rc3
-04f714765c13824c3bc2835d7b008908862e083a v3.4.0
-c67a19e11a7191baf30f313bf55e2e0b6c6f574e v3.4.1rc1
-c0e311e010fcb5bae8d87ca22051cd0845ea0ca0 v3.4.1
-8711a09513848cfc48c689d983495ee64f4668ca v3.4.2rc1
-ab2c023a9432f16652e89c404bbc84aa91bf55af v3.4.2
-69dd528ca6255a66c37cc5cf680e8357d108b036 v3.4.3rc1
-b4cbecbc0781e89a309d03b60a1f75f8499250e6 v3.4.3
-04f3f725896c6961212c3a12e8ac25be6958f4fa v3.4.4rc1
-737efcadf5a678b184e0fa431aae11276bf06648 v3.4.4
-3631bb4a2490292ebf81d3e947ae36da145da564 v3.4.5rc1
-619b61e505d0e2ccc8516b366e4ddd1971b46a6f v3.4.5
-e199a272ccdac5a8c073d4690f60c13e0b6d86b0 v3.4.6rc1
-b662f47769213f23325144b80782c05764d0f053 v3.4.6
-5d4b6a57d5fd7564bf73f3db0e46fe5eeb00bcd8 v3.5.0a1
-0337bd7ebcb6559d69679bc7025059ad1ce4f432 v3.5.0a2
-82656e28b5e5c4ae48d8dd8b5f0d7968908a82b6 v3.5.0a3
-413e0e0004f4f954331cb8122aa55fe208984955 v3.5.0a4
-071fefbb5e3db770c6c19fba9994699f121b1cea v3.5.0b1
-7a088af5615bf04024e9912068f4bd8f43ed3917 v3.5.0b2
-0035fcd9b9243ae52c2e830204fd9c1f7d528534 v3.5.0b3
-c0d64105463581f85d0e368e8d6e59b7fd8f12b1 v3.5.0b4
-1a58b1227501e046eee13d90f113417b60843301 v3.5.0rc1
-cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2
-66ed52375df802f9d0a34480daaa8ce79fc41313 v3.5.0rc3
-2d033fedfa7f1e325fd14ccdaa9cb42155da206f v3.5.0rc4
-374f501f4567b7595f2ad7798aa09afa2456bb28 v3.5.0
-948ef16a69513ba1ff15c9d7d0b012b949df4c80 v3.5.1rc1
-37a07cee5969e6d3672583187a73cf636ff28e1b v3.5.1
-68feec6488b26327a85a634605dd28eca4daa5f1 v3.5.2rc1
-4def2a2901a5618ea45bcc8f2a1411ef33af18ad v3.5.2
-de530d7f21c0398bb2a2b67716e0638e5fadf727 v3.5.3rc1
-1880cb95a742cd001c67677de5c4efeab169416c v3.5.3
-5896da372fb044e38595fb74495de1e1e7c8fb3c v3.6.0a1
-37889342355223e2fc1438de3dc7ffcd625c60f7 v3.6.0a2
-f3edf13dc339b8942ae6b309771ab197dd8ce6fa v3.6.0a3
-017cf260936b444788c9b671d195b7bfd83dbd25 v3.6.0a4
-5b0ca4ed5e2f0669d76ece7ef975c544580f12b4 v3.6.0b1
-b9fadc7d1c3f9c3c77f32f35afbe1a1cc38070e6 v3.6.0b2
-8345e066c0ed713c3e510cbc8fafc1c38d6d306b v3.6.0b3
-18496abdb3d5c2730a659b747a89261b2219fecf v3.6.0b4
-29a273eee9a523ee178f6a66c4ac9d317c8fc84f v3.6.0rc1
-800a67f7806de45a7abd5273359e704bf147c079 v3.6.0rc2
-41df79263a11f2429d1dd0cfe12553de3dcb5508 v3.6.0
diff --git a/.hgtouch b/.hgtouch
deleted file mode 100644
index b9be0f11fdb829f..000000000000000
--- a/.hgtouch
+++ /dev/null
@@ -1,17 +0,0 @@
-# -*- Makefile -*-
-# Define dependencies of generated files that are checked into hg.
-# The syntax of this file uses make rule dependencies, without actions
-
-Python/importlib.h: Lib/importlib/_bootstrap.py Programs/_freeze_importlib.c
-
-Include/opcode.h: Lib/opcode.py Tools/scripts/generate_opcode_h.py
-
-Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py
-Python/Python-ast.c: Include/Python-ast.h
-
-Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py
-
-Objects/typeslots.inc: Include/typeslots.h Objects/typeslots.py
-
-Include/graminit.h: Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c
-Python/graminit.c: Include/graminit.h Grammar/Grammar Parser/acceler.c Parser/grammar1.c Parser/listnode.c Parser/node.c Parser/parser.c Parser/bitset.c Parser/metagrammar.c Parser/firstsets.c Parser/grammar.c Parser/pgen.c Objects/obmalloc.c Python/dynamic_annotations.c Python/mysnprintf.c Python/pyctype.c Parser/tokenizer_pgen.c Parser/printgrammar.c Parser/parsetok_pgen.c Parser/pgenmain.c
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000000000..0c4c28e8295bc68
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,135 @@
+language: c
+dist: xenial
+
+# To cache doc-building dependencies and C compiler output.
+cache:
+ - pip
+ - ccache
+
+env:
+ global:
+ # Use -O3 because we don't use debugger on Travis-CI
+ - CFLAGS="-O3"
+
+branches:
+ only:
+ - master
+ - /^\d\.\d$/
+ - buildbot-custom
+
+matrix:
+ fast_finish: true
+ allow_failures:
+ - env: OPTIONAL=true
+ include:
+ - os: linux
+ language: c
+ compiler: clang
+ # gcc also works, but to keep the # of concurrent builds down, we use one C
+ # compiler here and the other to run the coverage build. Clang is preferred
+ # in this instance for its better error messages.
+ env: TESTING=cpython
+ addons:
+ apt:
+ packages:
+ - xvfb
+ - os: linux
+ language: python
+ python: 3.6
+ env: TESTING=docs
+ before_script:
+ - cd Doc
+ # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures.
+ # (Updating the version is fine as long as no warnings are raised by doing so.)
+ - python -m pip install sphinx==1.8.2 blurb
+ script:
+ - make check suspicious html SPHINXOPTS="-q -W -j4"
+ - os: linux
+ language: c
+ compiler: gcc
+ env: OPTIONAL=true
+ addons:
+ apt:
+ packages:
+ - xvfb
+ before_script:
+ - ./configure PYTHON_FOR_REGEN=python3
+ - make -s -j4
+ # Need a venv that can parse covered code.
+ - ./python -m venv venv
+ - ./venv/bin/python -m pip install -U coverage
+ - ./venv/bin/python -m test.pythoninfo
+ script:
+ # Skip tests that re-run the entire test suite.
+ - xvfb-run ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn
+ after_script: # Probably should be after_success once test suite updated to run under coverage.py.
+ # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
+ - source ./venv/bin/activate
+ - bash <(curl -s https://codecov.io/bash)
+
+
+before_install:
+ - set -e
+ - |
+ # Check short-circuit conditions
+ if [ "${TESTING}" != "docs" ]
+ then
+ if [ "$TRAVIS_PULL_REQUEST" = "false" ]
+ then
+ echo "Not a PR, doing full build."
+ else
+ # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE
+ # may include more changes than desired if the history is convoluted.
+ # Instead, explicitly fetch the base branch and compare against the
+ # merge-base commit.
+ git fetch -q origin +refs/heads/$TRAVIS_BRANCH
+ changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD))
+ echo "Files changed:"
+ echo "$changes"
+ if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)'
+ then
+ echo "Only docs were updated, stopping build process."
+ exit
+ fi
+ fi
+ fi
+
+# Travis provides only 2 cores, so don't overdo the parallelism and waste memory.
+before_script:
+ - ./configure --with-pydebug PYTHON_FOR_REGEN=python3
+ - make -j4 regen-all
+ - changes=`git status --porcelain`
+ - |
+ # Check for changes in regenerated files
+ if ! test -z "$changes"
+ then
+ echo "Generated files not up to date"
+ echo "$changes"
+ exit 1
+ fi
+ - make -j4
+ - make pythoninfo
+
+script:
+ # Using the built Python as patchcheck.py is built around the idea of using
+ # a checkout-build of CPython to know things like what base branch the changes
+ # should be compared against.
+ # Only run on Linux as the check only needs to be run once.
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi
+ # Check that all symbols exported by libpython start with "Py" or "_Py"
+ - make smelly
+ # `-r -w` implicitly provided through `make buildbottest`.
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then XVFB_RUN=xvfb-run; fi; $XVFB_RUN make buildbottest TESTOPTS="-j4 -uall,-cpu"
+
+notifications:
+ email: false
+ irc:
+ channels:
+ # This is set to a secure variable to prevent forks from notifying the
+ # IRC channel whenever they fail a build. This can be removed when travis
+ # implements https://github.com/travis-ci/travis-ci/issues/1094.
+ # The actual value here is: irc.freenode.net#python-dev
+ - secure: "s7kAkpcom2yUJ8XqyjFI0obJmhAGrn1xmoivdaPdgBIA++X47TBp1x4pgDsbEsoalef7bEwa4l07KdT4qa+DOd/c4QxaWom7fbN3BuLVsZuVfODnl79+gYq/TAbGfyH+yDs18DXrUfPgwD7C5aW32ugsqAOd4iWzfGJQ5OrOZzqzGjYdYQUEkJFXgxDEIb4aHvxNDWGO3Po9uKISrhb5saQ0l776yLo1Ur7M4oxl8RTbCdgX0vf5TzPg52BgvZpOgt3DHOUYPeiJLKNjAE6ibg0U95sEvMfHX77nz4aFY4/3UI6FFaRla34rZ+mYKrn0TdxOhera1QOgPmM6HzdO4K44FpfK1DS0Xxk9U9/uApq+cG0bU3W+cVUHDBe5+90lpRBAXHeHCgT7TI8gec614aiT8lEr3+yH8OBRYGzkjNK8E2LJZ/SxnVxDe7aLF6AWcoWLfS6/ziAIBFQ5Nc4U72CT8fGVSkl8ywPiRlvixKdvTODMSZo0jMqlfZSNaAPTsNRx4wu5Uis4qekwe32Fz4aB6KGpsuuVjBi+H6v0RKxNJNGY3JKDiEH2TK0UE2auJ5GvLW48aUVFcQMB7euCWYXlSWVRHh3WLU8QXF29Dw4JduRZqUpOdRgMHU79UHRq+mkE0jAS/nBcS6CvsmxCpTSrfVYuMOu32yt18QQoTyU="
+ on_success: change
+ on_failure: always
+ skip_join: true
diff --git a/Doc/Makefile b/Doc/Makefile
index 91f937f985831af..dbefbb4c3c1cd6d 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -4,13 +4,15 @@
#
# You can set these variables from the command line.
-PYTHON = python
-SPHINXBUILD = sphinx-build
+PYTHON ?= python3
+VENVDIR = ./venv
+SPHINXBUILD ?= PATH=$(VENVDIR)/bin:$$PATH sphinx-build
+BLURB = $(VENVDIR)/bin/python3 $(VENVDIR)/blurb.py
PAPER =
SOURCES =
DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py)
-ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
+ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_elements.papersize=$(PAPER) \
$(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
.PHONY: help build html htmlhelp latex text changes linkcheck \
@@ -38,6 +40,20 @@ help:
@echo " serve to serve the documentation on the localhost (8000)"
build:
+ -mkdir -p build
+# Look first for a Misc/NEWS file (building from a source release tarball
+# or old repo) and use that, otherwise look for a Misc/NEWS.d directory
+# (building from a newer repo) and use blurb to generate the NEWS file.
+ @if [ -f ../Misc/NEWS ] ; then \
+ echo "Using existing Misc/NEWS file"; \
+ cp ../Misc/NEWS build/NEWS; \
+ elif [ -d ../Misc/NEWS.d ]; then \
+ echo "Building NEWS from Misc/NEWS.d with blurb"; \
+ $(BLURB) merge -f build/NEWS; \
+ else \
+ echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \
+ exit 1; \
+ fi
$(SPHINXBUILD) $(ALLSPHINXOPTS)
@echo
@@ -103,11 +119,15 @@ htmlview: html
$(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')"
clean:
- -rm -rf build/* venv/*
+ -rm -rf build/* $(VENVDIR)/*
venv:
- $(PYTHON) -m venv venv
- ./venv/bin/python3 -m pip install -U Sphinx
+ $(PYTHON) -m venv $(VENVDIR)
+ $(VENVDIR)/bin/python3 -m pip install -r requirements.txt
+ # Patch a private copy, leaving the installed package unchanged.
+ cp "$$($(VENVDIR)/bin/python3 -c 'import blurb; print(blurb.__file__)')" $(VENVDIR)/blurb.py
+ patch --batch $(VENVDIR)/blurb.py tools/blurb-gh-issues.patch
+ @echo "The venv has been created in the $(VENVDIR) directory"
dist:
rm -rf dist
@@ -153,21 +173,26 @@ dist:
cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
check:
- $(PYTHON) tools/rstlint.py -i tools -i venv
+ $(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst
serve:
../Tools/scripts/serve.py build/html
# Targets for daily automated doc build
+# By default, Sphinx only rebuilds pages where the page content has changed.
+# This means it doesn't always pick up changes to preferred link targets, etc
+# To ensure such changes are picked up, we build the published docs with
+# `-E` (to ignore the cached environment) and `-a` (to ignore already existing
+# output files)
# for development releases: always build
autobuild-dev:
- make dist SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1'
+ make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
-make suspicious
# for quick rebuilds (HTML only)
autobuild-dev-html:
- make html SPHINXOPTS='$(SPHINXOPTS) -A daily=1 -A versionswitcher=1'
+ make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
# for stable releases: only build if not in pre-release stage (alpha, beta)
# release candidate downloads are okay, since the stable tree can be in that stage
diff --git a/Doc/README.txt b/Doc/README.rst
similarity index 52%
rename from Doc/README.txt
rename to Doc/README.rst
index 4f8e9f8f1417fbc..19206c85b7cb137 100644
--- a/Doc/README.txt
+++ b/Doc/README.rst
@@ -2,49 +2,70 @@ Python Documentation README
~~~~~~~~~~~~~~~~~~~~~~~~~~~
This directory contains the reStructuredText (reST) sources to the Python
-documentation. You don't need to build them yourself, prebuilt versions are
-available at .
+documentation. You don't need to build them yourself, `prebuilt versions are
+available `_.
Documentation on authoring Python documentation, including information about
-both style and markup, is available in the "Documenting Python" chapter of the
-developers guide .
+both style and markup, is available in the "`Documenting Python
+`_" chapter of the
+developers guide.
Building the docs
=================
-You need to have Sphinx installed; it is the toolset
-used to build the docs. It is not included in this tree, but maintained
-separately and available from PyPI .
+The documentation is built with several tools which are not included in this
+tree but are maintained separately and are available from
+`PyPI `_.
+
+* `Sphinx `_
+* `blurb `_
+
+The easiest way to install these tools is to create a virtual environment and
+install the tools into there.
Using make
----------
-A Makefile has been prepared so that on Unix, provided you have installed
-Sphinx, you can just run ::
+To get started on UNIX, you can create a virtual environment with the command ::
- make html
+ make venv
-to build the HTML output files.
+That will install all the tools necessary to build the documentation. Assuming
+the virtual environment was created in the ``venv`` directory (the default;
+configurable with the VENVDIR variable), you can run the following command to
+build the HTML output files::
-On Windows, we try to emulate the Makefile as closely as possible with a
-``make.bat`` file.
+ make html
-To use a Python interpreter that's not called ``python``, use the standard
-way to set Makefile variables, using e.g. ::
+This branch pins the Python tools in ``requirements.txt`` for Python 3.6.
+The ``venv`` target also requires GNU ``patch``. It copies blurb 1.0.7 to
+``venv/blurb.py`` and applies ``tools/blurb-gh-issues.patch`` to that private
+copy so NEWS generation accepts both Roundup and GitHub issue numbers.
+The installed blurb package and NEWS fragments are not modified. Repeating
+``make venv`` recreates the patched copy from the installed package.
- make html PYTHON=python3
+The Makefile looks for Sphinx in the virtual environment and then on PATH.
+It uses the private blurb copy by default. Override ``SPHINXBUILD`` or ``BLURB``
+to use other tools; an external blurb must support GitHub issue entries.
-On Windows, set the PYTHON environment variable instead.
+To run the documentation-tool regression tests::
-To use a specific sphinx-build (something other than ``sphinx-build``), set
-the SPHINXBUILD variable.
+ venv/bin/python3 tools/test_blurb.py
+ venv/bin/python3 tools/test_extensions.py
+
+On Windows, we try to emulate the Makefile as closely as possible with a
+``make.bat`` file. If you need to specify the Python interpreter to use,
+set the PYTHON environment variable instead.
Available make targets are:
* "clean", which removes all build files.
+* "venv", which creates a virtual environment with all necessary tools
+ installed.
+
* "html", which builds standalone HTML files for offline viewing.
* "htmlview", which re-uses the "html" builder, but then opens the main page
@@ -95,7 +116,7 @@ Available make targets are:
Without make
------------
-Install the Sphinx package and its dependencies from PyPI.
+First, install the tool dependencies from PyPI.
Then, from the ``Doc`` directory, run ::
@@ -104,15 +125,23 @@ Then, from the ``Doc`` directory, run ::
where ```` is one of html, text, latex, or htmlhelp (for explanations
see the make targets above).
+Deprecation header
+==================
+
+You can define the ``outdated`` variable in ``html_context`` to show a
+red banner on each page redirecting to the "latest" version.
+
+The link points to the same page on ``/3/``, sadly for the moment the
+language is lost during the process.
+
Contributing
============
-Bugs in the content should be reported to the Python bug tracker at
-https://bugs.python.org.
+Bugs in the content should be reported to the
+`Python bug tracker `_.
-Bugs in the toolset should be reported in the Sphinx bug tracker at
-https://www.bitbucket.org/birkenfeld/sphinx/issues/.
+Bugs in the toolset should be reported to the tools themselves.
You can also send a mail to the Python Documentation Team at docs@python.org,
and we will process your request as soon as possible.
diff --git a/Doc/bugs.rst b/Doc/bugs.rst
index 1b0a5a9a9377cbe..bc1d10f379cb350 100644
--- a/Doc/bugs.rst
+++ b/Doc/bugs.rst
@@ -88,5 +88,5 @@ the `core-mentorship mailing list`_ is a friendly place to get answers to
any and all questions pertaining to the process of fixing issues in Python.
.. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity
-.. _Python Developer's Guide: https://docs.python.org/devguide/
+.. _Python Developer's Guide: https://devguide.python.org/
.. _core-mentorship mailing list: https://mail.python.org/mailman/listinfo/core-mentorship/
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index e8e7e0d559309e3..dc58e152a6b56f3 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -151,19 +151,35 @@ which disallows mutable objects such as :class:`bytearray`.
Previously, :exc:`TypeError` was raised when embedded null code points
were encountered in the Python string.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``u#`` (:class:`str`) [Py_UNICODE \*, int]
This variant on ``u`` stores into two C variables, the first one a pointer to a
Unicode data buffer, the second one its length. This variant allows
null code points.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``Z`` (:class:`str` or ``None``) [Py_UNICODE \*]
Like ``u``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``Z#`` (:class:`str` or ``None``) [Py_UNICODE \*, int]
Like ``u#``, but the Python object may also be ``None``, in which case the
:c:type:`Py_UNICODE` pointer is set to *NULL*.
+ .. deprecated-removed:: 3.3 4.0
+ Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
+ :c:func:`PyUnicode_AsWideCharString`.
+
``U`` (:class:`str`) [PyObject \*]
Requires that the Python object is a Unicode object, without attempting
any conversion. Raises :exc:`TypeError` if the object is not a Unicode
@@ -552,12 +568,13 @@ Building values
``z#`` (:class:`str` or ``None``) [char \*, int]
Same as ``s#``.
- ``u`` (:class:`str`) [Py_UNICODE \*]
- Convert a null-terminated buffer of Unicode (UCS-2 or UCS-4) data to a Python
- Unicode object. If the Unicode buffer pointer is *NULL*, ``None`` is returned.
+ ``u`` (:class:`str`) [wchar_t \*]
+ Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
+ data to a Python Unicode object. If the Unicode buffer pointer is *NULL*,
+ ``None`` is returned.
- ``u#`` (:class:`str`) [Py_UNICODE \*, int]
- Convert a Unicode (UCS-2 or UCS-4) data buffer and its length to a Python
+ ``u#`` (:class:`str`) [wchar_t \*, int]
+ Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python
Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored
and ``None`` is returned.
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index 3d851b7c39ce0ff..c7c1e3cc745ac40 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -198,7 +198,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`.
indicates that no de-referencing should occur (striding in a contiguous
memory block).
- If all suboffsets are negative (i.e. no de-referencing is needed, then
+ If all suboffsets are negative (i.e. no de-referencing is needed), then
this field must be NULL (the default value).
This type of array representation is used by the Python Imaging Library
@@ -429,7 +429,7 @@ Buffer-related functions
Return ``1`` if *obj* supports the buffer interface otherwise ``0``. When ``1`` is
returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will
- succeed.
+ succeed. This function always succeeds.
.. c:function:: int PyObject_GetBuffer(PyObject *exporter, Py_buffer *view, int flags)
@@ -470,10 +470,19 @@ Buffer-related functions
Return ``1`` if the memory defined by the *view* is C-style (*order* is
``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one
- (*order* is ``'A'``). Return ``0`` otherwise.
+ (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds.
-.. c:function:: void PyBuffer_FillContiguousStrides(int ndim, Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t itemsize, char order)
+.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order)
+
+ Copy *len* bytes from *src* to its contiguous representation in *buf*.
+ *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering).
+ ``0`` is returned on success, ``-1`` on error.
+
+ This function fails if *len* != *src->len*.
+
+
+.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order)
Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if
*order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the
@@ -497,6 +506,3 @@ Buffer-related functions
If this function is used as part of a :ref:`getbufferproc `,
*exporter* MUST be set to the exporting object and *flags* must be passed
unmodified. Otherwise, *exporter* MUST be NULL.
-
-
-
diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst
index b8642d0aba9248a..8eb6695e22de189 100644
--- a/Doc/c-api/capsule.rst
+++ b/Doc/c-api/capsule.rst
@@ -9,6 +9,8 @@ Capsules
Refer to :ref:`using-capsules` for more information on using these objects.
+.. versionadded:: 3.1
+
.. c:type:: PyCapsule
@@ -19,6 +21,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
regular import mechanism can be used to access C APIs defined in dynamically
loaded modules.
+
.. c:type:: PyCapsule_Destructor
The type of a destructor callback for a capsule. Defined as::
@@ -104,8 +107,8 @@ Refer to :ref:`using-capsules` for more information on using these objects.
import the module conventionally (using :c:func:`PyImport_ImportModule`).
Return the capsule's internal *pointer* on success. On failure, set an
- exception and return *NULL*. However, if :c:func:`PyCapsule_Import` failed to
- import the module, and *no_block* was true, no exception is set.
+ exception and return *NULL*.
+
.. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name)
@@ -122,18 +125,21 @@ Refer to :ref:`using-capsules` for more information on using these objects.
Return a nonzero value if the object is valid and matches the name passed in.
Return ``0`` otherwise. This function will not fail.
+
.. c:function:: int PyCapsule_SetContext(PyObject *capsule, void *context)
Set the context pointer inside *capsule* to *context*.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetDestructor(PyObject *capsule, PyCapsule_Destructor destructor)
Set the destructor inside *capsule* to *destructor*.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name)
Set the name inside *capsule* to *name*. If non-*NULL*, the name must
@@ -142,6 +148,7 @@ Refer to :ref:`using-capsules` for more information on using these objects.
Return ``0`` on success. Return nonzero and set an exception on failure.
+
.. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer)
Set the void pointer inside *capsule* to *pointer*. The pointer may not be
diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst
index dfe3d436e5f4b7c..c55f19970e125dc 100644
--- a/Doc/c-api/codec.rst
+++ b/Doc/c-api/codec.rst
@@ -13,7 +13,7 @@ Codec registry and support functions
.. c:function:: int PyCodec_KnownEncoding(const char *encoding)
Return ``1`` or ``0`` depending on whether there is a registered codec for
- the given *encoding*.
+ the given *encoding*. This function always succeeds.
.. c:function:: PyObject* PyCodec_Encode(PyObject *object, const char *encoding, const char *errors)
diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst
index 39542bd17a3b4a7..305e990368c2931 100644
--- a/Doc/c-api/datetime.rst
+++ b/Doc/c-api/datetime.rst
@@ -188,7 +188,7 @@ not be *NULL*, and the type is not checked:
.. versionadded:: 3.3
-.. c:function:: int PyDateTime_DELTA_GET_MICROSECOND(PyDateTime_Delta *o)
+.. c:function:: int PyDateTime_DELTA_GET_MICROSECONDS(PyDateTime_Delta *o)
Return the number of microseconds, as an int from 0 through 999999.
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index cfa5e13b9feef8a..68ca25ca548d60d 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -95,6 +95,10 @@ Dictionary Objects
Return the object from dictionary *p* which has a key *key*. Return *NULL*
if the key *key* is not present, but *without* setting an exception.
+ Note that exceptions which occur while calling :meth:`__hash__` and
+ :meth:`__eq__` methods will get suppressed.
+ To get error reporting use :c:func:`PyDict_GetItemWithError()` instead.
+
.. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key)
@@ -109,6 +113,11 @@ Dictionary Objects
This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a
:c:type:`char\*`, rather than a :c:type:`PyObject\*`.
+ Note that exceptions which occur while calling :meth:`__hash__` and
+ :meth:`__eq__` methods and creating a temporary string object
+ will get suppressed.
+ To get error reporting use :c:func:`PyDict_GetItemWithError()` instead.
+
.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default)
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 037b85cfd117542..6ceb8eab62d497f 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -53,8 +53,12 @@ Printing and clearing
.. c:function:: void PyErr_PrintEx(int set_sys_last_vars)
Print a standard traceback to ``sys.stderr`` and clear the error indicator.
- Call this function only when the error indicator is set. (Otherwise it will
- cause a fatal error!)
+ **Unless** the error is a ``SystemExit``. In that case the no traceback
+ is printed and Python process will exit with the error code specified by
+ the ``SystemExit`` instance.
+
+ Call this function **only** when the error indicator is set. Otherwise it
+ will cause a fatal error!
If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`,
:data:`sys.last_value` and :data:`sys.last_traceback` will be set to the
@@ -291,16 +295,11 @@ an error value).
is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that,
and so forth.
- Warning categories must be subclasses of :c:data:`Warning`; the default warning
- category is :c:data:`RuntimeWarning`. The standard Python warning categories are
- available as global variables whose names are ``PyExc_`` followed by the Python
- exception name. These have the type :c:type:`PyObject\*`; they are all class
- objects. Their names are :c:data:`PyExc_Warning`, :c:data:`PyExc_UserWarning`,
- :c:data:`PyExc_UnicodeWarning`, :c:data:`PyExc_DeprecationWarning`,
- :c:data:`PyExc_SyntaxWarning`, :c:data:`PyExc_RuntimeWarning`, and
- :c:data:`PyExc_FutureWarning`. :c:data:`PyExc_Warning` is a subclass of
- :c:data:`PyExc_Exception`; the other warning categories are subclasses of
- :c:data:`PyExc_Warning`.
+ Warning categories must be subclasses of :c:data:`PyExc_Warning`;
+ :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`;
+ the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard
+ Python warning categories are available as global variables whose names are
+ enumerated at :ref:`standardwarningcategories`.
For information about warning control, see the documentation for the
:mod:`warnings` module and the :option:`-W` option in the command line
@@ -505,13 +504,13 @@ Signal Handling
single: SIGINT
single: KeyboardInterrupt (built-in exception)
- This function simulates the effect of a :const:`SIGINT` signal arriving --- the
- next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will
- be raised. It may be called without holding the interpreter lock.
-
- .. % XXX This was described as obsolete, but is used in
- .. % _thread.interrupt_main() (used from IDLE), so it's still needed.
+ Simulate the effect of a :const:`SIGINT` signal arriving. The next time
+ :c:func:`PyErr_CheckSignals` is called, the Python signal handler for
+ :const:`SIGINT` will be called.
+ If :const:`SIGINT` isn't handled by Python (it was set to
+ :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+ nothing.
.. c:function:: int PySignal_SetWakeupFd(int fd)
@@ -750,6 +749,61 @@ All standard Python exceptions are available as global variables whose names are
:c:type:`PyObject\*`; they are all class objects. For completeness, here are all
the variables:
+.. index::
+ single: PyExc_BaseException
+ single: PyExc_Exception
+ single: PyExc_ArithmeticError
+ single: PyExc_AssertionError
+ single: PyExc_AttributeError
+ single: PyExc_BlockingIOError
+ single: PyExc_BrokenPipeError
+ single: PyExc_BufferError
+ single: PyExc_ChildProcessError
+ single: PyExc_ConnectionAbortedError
+ single: PyExc_ConnectionError
+ single: PyExc_ConnectionRefusedError
+ single: PyExc_ConnectionResetError
+ single: PyExc_EOFError
+ single: PyExc_FileExistsError
+ single: PyExc_FileNotFoundError
+ single: PyExc_FloatingPointError
+ single: PyExc_GeneratorExit
+ single: PyExc_ImportError
+ single: PyExc_IndentationError
+ single: PyExc_IndexError
+ single: PyExc_InterruptedError
+ single: PyExc_IsADirectoryError
+ single: PyExc_KeyError
+ single: PyExc_KeyboardInterrupt
+ single: PyExc_LookupError
+ single: PyExc_MemoryError
+ single: PyExc_ModuleNotFoundError
+ single: PyExc_NameError
+ single: PyExc_NotADirectoryError
+ single: PyExc_NotImplementedError
+ single: PyExc_OSError
+ single: PyExc_OverflowError
+ single: PyExc_PermissionError
+ single: PyExc_ProcessLookupError
+ single: PyExc_RecursionError
+ single: PyExc_ReferenceError
+ single: PyExc_RuntimeError
+ single: PyExc_StopAsyncIteration
+ single: PyExc_StopIteration
+ single: PyExc_SyntaxError
+ single: PyExc_SystemError
+ single: PyExc_SystemExit
+ single: PyExc_TabError
+ single: PyExc_TimeoutError
+ single: PyExc_TypeError
+ single: PyExc_UnboundLocalError
+ single: PyExc_UnicodeDecodeError
+ single: PyExc_UnicodeEncodeError
+ single: PyExc_UnicodeError
+ single: PyExc_UnicodeTranslateError
+ single: PyExc_ValueError
+ single: PyExc_ZeroDivisionError
+
+-----------------------------------------+---------------------------------+----------+
| C Name | Python Name | Notes |
+=========================================+=================================+==========+
@@ -759,8 +813,6 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
-+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | |
@@ -769,27 +821,31 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | |
+| :c:data:`PyExc_BufferError` | :exc:`BufferError` | |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | |
+| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_EOFError` | :exc:`EOFError` | |
-+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ImportError` | :exc:`ImportError` | |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | |
+| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_IndexError` | :exc:`IndexError` | |
+-----------------------------------------+---------------------------------+----------+
@@ -801,8 +857,12 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_NameError` | :exc:`NameError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | |
@@ -823,16 +883,32 @@ the variables:
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_SystemError` | :exc:`SystemError` | |
+-----------------------------------------+---------------------------------+----------+
-| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | |
-+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TabError` | :exc:`TabError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_TypeError` | :exc:`TypeError` | |
+-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | |
++-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | |
++-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ValueError` | :exc:`ValueError` | |
+-----------------------------------------+---------------------------------+----------+
| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | |
@@ -849,11 +925,18 @@ the variables:
and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`.
.. versionadded:: 3.5
- :c:data:`PyExc_RecursionError`.
+ :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`.
+.. versionadded:: 3.6
+ :c:data:`PyExc_ModuleNotFoundError`.
These are compatibility aliases to :c:data:`PyExc_OSError`:
+.. index::
+ single: PyExc_EnvironmentError
+ single: PyExc_IOError
+ single: PyExc_WindowsError
+
+-------------------------------------+----------+
| C Name | Notes |
+=====================================+==========+
@@ -867,52 +950,6 @@ These are compatibility aliases to :c:data:`PyExc_OSError`:
.. versionchanged:: 3.3
These aliases used to be separate exception types.
-
-.. index::
- single: PyExc_BaseException
- single: PyExc_Exception
- single: PyExc_ArithmeticError
- single: PyExc_LookupError
- single: PyExc_AssertionError
- single: PyExc_AttributeError
- single: PyExc_BlockingIOError
- single: PyExc_BrokenPipeError
- single: PyExc_ConnectionError
- single: PyExc_ConnectionAbortedError
- single: PyExc_ConnectionRefusedError
- single: PyExc_ConnectionResetError
- single: PyExc_EOFError
- single: PyExc_FileExistsError
- single: PyExc_FileNotFoundError
- single: PyExc_FloatingPointError
- single: PyExc_ImportError
- single: PyExc_IndexError
- single: PyExc_InterruptedError
- single: PyExc_IsADirectoryError
- single: PyExc_KeyError
- single: PyExc_KeyboardInterrupt
- single: PyExc_MemoryError
- single: PyExc_NameError
- single: PyExc_NotADirectoryError
- single: PyExc_NotImplementedError
- single: PyExc_OSError
- single: PyExc_OverflowError
- single: PyExc_PermissionError
- single: PyExc_ProcessLookupError
- single: PyExc_RecursionError
- single: PyExc_ReferenceError
- single: PyExc_RuntimeError
- single: PyExc_SyntaxError
- single: PyExc_SystemError
- single: PyExc_SystemExit
- single: PyExc_TimeoutError
- single: PyExc_TypeError
- single: PyExc_ValueError
- single: PyExc_ZeroDivisionError
- single: PyExc_EnvironmentError
- single: PyExc_IOError
- single: PyExc_WindowsError
-
Notes:
(1)
@@ -924,3 +961,60 @@ Notes:
(3)
Only defined on Windows; protect code that uses this by testing that the
preprocessor macro ``MS_WINDOWS`` is defined.
+
+.. _standardwarningcategories:
+
+Standard Warning Categories
+===========================
+
+All standard Python warning categories are available as global variables whose
+names are ``PyExc_`` followed by the Python exception name. These have the type
+:c:type:`PyObject\*`; they are all class objects. For completeness, here are all
+the variables:
+
+.. index::
+ single: PyExc_Warning
+ single: PyExc_BytesWarning
+ single: PyExc_DeprecationWarning
+ single: PyExc_FutureWarning
+ single: PyExc_ImportWarning
+ single: PyExc_PendingDeprecationWarning
+ single: PyExc_ResourceWarning
+ single: PyExc_RuntimeWarning
+ single: PyExc_SyntaxWarning
+ single: PyExc_UnicodeWarning
+ single: PyExc_UserWarning
+
++------------------------------------------+---------------------------------+----------+
+| C Name | Python Name | Notes |
++==========================================+=================================+==========+
+| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | |
++------------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | |
++------------------------------------------+---------------------------------+----------+
+
+.. versionadded:: 3.2
+ :c:data:`PyExc_ResourceWarning`.
+
+Notes:
+
+(1)
+ This is a base class for other standard warning categories.
diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst
index f5e0d7ec9c79c36..472cd93ec3f7e19 100644
--- a/Doc/c-api/gcsupport.rst
+++ b/Doc/c-api/gcsupport.rst
@@ -49,7 +49,7 @@ Constructors for container types must conform to two rules:
.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize)
Resize an object allocated by :c:func:`PyObject_NewVar`. Returns the
- resized object or *NULL* on failure.
+ resized object or *NULL* on failure. *op* must not be tracked by the collector yet.
.. c:function:: void PyObject_GC_Track(PyObject *op)
@@ -66,6 +66,9 @@ Constructors for container types must conform to two rules:
A macro version of :c:func:`PyObject_GC_Track`. It should not be used for
extension modules.
+ .. deprecated:: 3.6
+ This macro is removed from Python 3.8.
+
Similarly, the deallocator for the object must conform to a similar pair of
rules:
@@ -95,6 +98,9 @@ rules:
A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for
extension modules.
+ .. deprecated:: 3.6
+ This macro is removed from Python 3.8.
+
The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type:
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 2965bc931a62535..f601476211411a1 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -64,8 +64,8 @@ Initializing and finalizing the interpreter
the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory
allocated by the Python interpreter. This is a no-op when called for a second
time (without calling :c:func:`Py_Initialize` again first). Normally the
- return value is 0. If there were errors during finalization
- (flushing buffered data), -1 is returned.
+ return value is ``0``. If there were errors during finalization
+ (flushing buffered data), ``-1`` is returned.
This function is provided for a number of reasons. An embedding application
might want to restart Python without having to restart the application itself.
@@ -1063,12 +1063,12 @@ Python-level trace functions in previous versions.
+------------------------------+--------------------------------------+
| Value of *what* | Meaning of *arg* |
+==============================+======================================+
- | :const:`PyTrace_CALL` | Always *NULL*. |
+ | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. |
+------------------------------+--------------------------------------+
| :const:`PyTrace_EXCEPTION` | Exception information as returned by |
| | :func:`sys.exc_info`. |
+------------------------------+--------------------------------------+
- | :const:`PyTrace_LINE` | Always *NULL*. |
+ | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. |
+------------------------------+--------------------------------------+
| :const:`PyTrace_RETURN` | Value being returned to the caller, |
| | or *NULL* if caused by an exception. |
@@ -1110,7 +1110,7 @@ Python-level trace functions in previous versions.
.. c:var:: int PyTrace_RETURN
The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a
- call is returning without propagating an exception.
+ call is about to return.
.. c:var:: int PyTrace_C_CALL
@@ -1137,15 +1137,19 @@ Python-level trace functions in previous versions.
function as its first parameter, and may be any Python object, or *NULL*. If
the profile function needs to maintain state, using a different value for *obj*
for each thread provides a convenient and thread-safe place to store it. The
- profile function is called for all monitored events except the line-number
- events.
+ profile function is called for all monitored events except :const:`PyTrace_LINE`
+ and :const:`PyTrace_EXCEPTION`.
.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj)
Set the tracing function to *func*. This is similar to
:c:func:`PyEval_SetProfile`, except the tracing function does receive line-number
- events.
+ events and does not receive any event related to C function objects being called. Any
+ trace function registered using :c:func:`PyEval_SetTrace` will not receive
+ :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN`
+ as a value for the *what* parameter.
+
.. c:function:: PyObject* PyEval_GetCallStats(PyObject *self)
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index f592cb65c3e3a91..5b1f386fb7e58fa 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -10,6 +10,9 @@ Integer Objects
All integers are implemented as "long" integer objects of arbitrary size.
+On error, most ``PyLong_As*`` APIs return ``(return type)-1`` which cannot be
+distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:type:: PyLongObject
This subtype of :c:type:`PyObject` represents a Python integer object.
@@ -85,13 +88,12 @@ All integers are implemented as "long" integer objects of arbitrary size.
Return a new :c:type:`PyLongObject` based on the string value in *str*, which
is interpreted according to the radix in *base*. If *pend* is non-*NULL*,
*\*pend* will point to the first character in *str* which follows the
- representation of the number. If *base* is ``0``, the radix will be
- determined based on the leading characters of *str*: if *str* starts with
- ``'0x'`` or ``'0X'``, radix 16 will be used; if *str* starts with ``'0o'`` or
- ``'0O'``, radix 8 will be used; if *str* starts with ``'0b'`` or ``'0B'``,
- radix 2 will be used; otherwise radix 10 will be used. If *base* is not
- ``0``, it must be between ``2`` and ``36``, inclusive. Leading spaces are
- ignored. If there are no digits, :exc:`ValueError` will be raised.
+ representation of the number. If *base* is ``0``, *str* is interpreted using
+ the :ref:`integers` definition; in this case, leading zeros in a
+ non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``,
+ it must be between ``2`` and ``36``, inclusive. Leading spaces and single
+ underscores after a base specifier and between digits are ignored. If there
+ are no digits, :exc:`ValueError` will be raised.
.. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base)
@@ -135,6 +137,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
:c:type:`long`.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow)
@@ -147,6 +151,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception
occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: long long PyLong_AsLongLong(PyObject *obj)
@@ -160,6 +166,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *obj* is out of range for a
:c:type:`long`.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow)
@@ -172,6 +180,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other
exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. versionadded:: 3.2
@@ -187,6 +197,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`Py_ssize_t`.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong)
@@ -200,15 +212,25 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`unsigned long`.
+ Returns ``(unsigned long)-1`` on error.
+ Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: size_t PyLong_AsSize_t(PyObject *pylong)
+ .. index::
+ single: SIZE_MAX
+ single: OverflowError (built-in exception)
+
Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be
an instance of :c:type:`PyLongObject`.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`size_t`.
+ Returns ``(size_t)-1`` on error.
+ Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: unsigned long long PyLong_AsUnsignedLongLong(PyObject *pylong)
@@ -221,6 +243,9 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for an
:c:type:`unsigned long long`.
+ Returns ``(unsigned long long)-1`` on error.
+ Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. versionchanged:: 3.1
A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`.
@@ -234,6 +259,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
If the value of *obj* is out of range for an :c:type:`unsigned long`,
return the reduction of that value modulo ``ULONG_MAX + 1``.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj)
@@ -244,6 +271,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
If the value of *obj* is out of range for an :c:type:`unsigned long long`,
return the reduction of that value modulo ``PY_ULLONG_MAX + 1``.
+ Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: double PyLong_AsDouble(PyObject *pylong)
@@ -253,6 +282,8 @@ All integers are implemented as "long" integer objects of arbitrary size.
Raise :exc:`OverflowError` if the value of *pylong* is out of range for a
:c:type:`double`.
+ Returns ``-1.0`` on error. Use :c:func:`PyErr_Occurred` to disambiguate.
+
.. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong)
@@ -260,3 +291,5 @@ All integers are implemented as "long" integer objects of arbitrary size.
If *pylong* cannot be converted, an :exc:`OverflowError` will be raised. This
is only assured to produce a usable :c:type:`void` pointer for values created
with :c:func:`PyLong_FromVoidPtr`.
+
+ Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate.
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index a71e94283776e96..cb68ef82ca83448 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -5,11 +5,17 @@
Mapping Protocol
================
+See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and
+:c:func:`PyObject_DelItem`.
+
.. c:function:: int PyMapping_Check(PyObject *o)
- Return ``1`` if the object provides mapping protocol, and ``0`` otherwise. This
- function always succeeds.
+ Return ``1`` if the object provides mapping protocol or supports slicing,
+ and ``0`` otherwise. Note that it returns ``1`` for Python classes with
+ a :meth:`__getitem__` method since in general case it is impossible to
+ determine what the type of keys it supports. This function always
+ succeeds.
.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o)
@@ -17,35 +23,57 @@ Mapping Protocol
.. index:: builtin: len
- Returns the number of keys in object *o* on success, and ``-1`` on failure. For
- objects that do not provide mapping protocol, this is equivalent to the Python
- expression ``len(o)``.
+ Returns the number of keys in object *o* on success, and ``-1`` on failure.
+ This is equivalent to the Python expression ``len(o)``.
-.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
+.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
- Remove the mapping for object *key* from the object *o*. Return ``-1`` on
- failure. This is equivalent to the Python statement ``del o[key]``.
+ Return element of *o* corresponding to the string *key* or *NULL* on failure.
+ This is the equivalent of the Python expression ``o[key]``.
+ See also :c:func:`PyObject_GetItem`.
+
+
+.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
+
+ Map the string *key* to the value *v* in object *o*. Returns ``-1`` on
+ failure. This is the equivalent of the Python statement ``o[key] = v``.
+ See also :c:func:`PyObject_SetItem`.
.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key)
- Remove the mapping for object *key* from the object *o*. Return ``-1`` on
- failure. This is equivalent to the Python statement ``del o[key]``.
+ Remove the mapping for the object *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
+ This is an alias of :c:func:`PyObject_DelItem`.
-.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
- On success, return ``1`` if the mapping object has the key *key* and ``0``
- otherwise. This is equivalent to the Python expression ``key in o``.
- This function always succeeds.
+ Remove the mapping for the string *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key)
- Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This
- is equivalent to the Python expression ``key in o``. This function always
- succeeds.
+ Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
+ This is equivalent to the Python expression ``key in o``.
+ This function always succeeds.
+
+ Note that exceptions which occur while calling the :meth:`__getitem__`
+ method will get suppressed.
+ To get error reporting use :c:func:`PyObject_GetItem()` instead.
+
+
+.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key)
+
+ Return ``1`` if the mapping object has the key *key* and ``0`` otherwise.
+ This is equivalent to the Python expression ``key in o``.
+ This function always succeeds.
+
+ Note that exceptions which occur while calling the :meth:`__getitem__`
+ method and creating a temporary string object will get suppressed.
+ To get error reporting use :c:func:`PyMapping_GetItemString()` instead.
.. c:function:: PyObject* PyMapping_Keys(PyObject *o)
@@ -64,15 +92,3 @@ Mapping Protocol
On success, return a list or tuple of the items in object *o*, where each item
is a tuple containing a key-value pair. On failure, return *NULL*.
-
-
-.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
-
- Return element of *o* corresponding to the object *key* or *NULL* on failure.
- This is the equivalent of the Python expression ``o[key]``.
-
-
-.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
-
- Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure.
- This is the equivalent of the Python statement ``o[key] = v``.
diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst
index a6d0f4688d1b78b..17ec621610b5e3a 100644
--- a/Doc/c-api/marshal.rst
+++ b/Doc/c-api/marshal.rst
@@ -34,18 +34,12 @@ unmarshalling. Version 2 uses a binary format for floating point numbers.
.. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version)
- Return a string object containing the marshalled representation of *value*.
+ Return a bytes object containing the marshalled representation of *value*.
*version* indicates the file format.
The following functions allow marshalled values to be read back in.
-XXX What about error detection? It appears that reading past the end of the
-file will always result in a negative numeric value (where that's relevant),
-but it's not clear that negative values won't be handled properly when there's
-no error. What's the right way to tell? Should only non-negative values be
-written using these routines?
-
.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file)
@@ -53,7 +47,8 @@ written using these routines?
for reading. Only a 32-bit value can be read in using this function,
regardless of the native size of :c:type:`long`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file)
@@ -62,7 +57,8 @@ written using these routines?
for reading. Only a 16-bit value can be read in using this function,
regardless of the native size of :c:type:`short`.
- On error, raise an exception and return ``-1``.
+ On error, sets the appropriate exception (:exc:`EOFError`) and returns
+ ``-1``.
.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file)
@@ -70,8 +66,8 @@ written using these routines?
Return a Python object from the data stream in a :c:type:`FILE\*` opened for
reading.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file)
@@ -84,15 +80,15 @@ written using these routines?
file. Only use these variant if you are certain that you won't be reading
anything else from the file.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
-.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *string, Py_ssize_t len)
+.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len)
- Return a Python object from the data stream in a character buffer
- containing *len* bytes pointed to by *string*.
+ Return a Python object from the data stream in a byte buffer
+ containing *len* bytes pointed to by *data*.
- On error, sets the appropriate exception (:exc:`EOFError` or
- :exc:`TypeError`) and returns *NULL*.
+ On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError`
+ or :exc:`TypeError`) and returns *NULL*.
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 3ff545275fb3b8a..b502b25f708bb5d 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -35,7 +35,7 @@ operate within the bounds of the private heap.
It is important to understand that the management of the Python heap is
performed by the interpreter itself and that the user has no control over it,
-even if she regularly manipulates object pointers to memory blocks inside that
+even if they regularly manipulate object pointers to memory blocks inside that
heap. The allocation of heap space for Python objects and other internal
buffers is performed on demand by the Python memory manager through the Python/C
API functions listed in this document.
@@ -150,7 +150,7 @@ The default raw memory block allocator uses the following functions:
Frees the memory block pointed to by *p*, which must have been returned by a
previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or
- :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_Free(p)`` has been
+ :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been
called before, undefined behavior occurs.
If *p* is *NULL*, no operation is performed.
@@ -263,6 +263,69 @@ versions and is therefore deprecated in extension modules.
* ``PyMem_DEL(ptr)``
+Object allocators
+=================
+
+The following function sets, modeled after the ANSI C standard, but specifying
+behavior when requesting zero bytes, are available for allocating and releasing
+memory from the Python heap.
+
+By default, these functions use :ref:`pymalloc memory allocator `.
+
+.. warning::
+
+ The :term:`GIL ` must be held when using these
+ functions.
+
+.. c:function:: void* PyObject_Malloc(size_t n)
+
+ Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the
+ allocated memory, or *NULL* if the request fails.
+
+ Requesting zero bytes returns a distinct non-*NULL* pointer if possible, as
+ if ``PyObject_Malloc(1)`` had been called instead. The memory will not have
+ been initialized in any way.
+
+
+.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize)
+
+ Allocates *nelem* elements each whose size in bytes is *elsize* and returns
+ a pointer of type :c:type:`void\*` to the allocated memory, or *NULL* if the
+ request fails. The memory is initialized to zeros.
+
+ Requesting zero elements or elements of size zero bytes returns a distinct
+ non-*NULL* pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called
+ instead.
+
+ .. versionadded:: 3.5
+
+
+.. c:function:: void* PyObject_Realloc(void *p, size_t n)
+
+ Resizes the memory block pointed to by *p* to *n* bytes. The contents will be
+ unchanged to the minimum of the old and the new sizes.
+
+ If *p* is *NULL*, the call is equivalent to ``PyObject_Malloc(n)``; else if *n*
+ is equal to zero, the memory block is resized but is not freed, and the
+ returned pointer is non-*NULL*.
+
+ Unless *p* is *NULL*, it must have been returned by a previous call to
+ :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`.
+
+ If the request fails, :c:func:`PyObject_Realloc` returns *NULL* and *p* remains
+ a valid pointer to the previous memory area.
+
+
+.. c:function:: void PyObject_Free(void *p)
+
+ Frees the memory block pointed to by *p*, which must have been returned by a
+ previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or
+ :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called
+ before, undefined behavior occurs.
+
+ If *p* is *NULL*, no operation is performed.
+
+
Customize Memory Allocators
===========================
@@ -391,7 +454,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and
:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex:
-:c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex:
+:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex:
:c:func:`PyObject_Malloc`) domains.
The arena allocator uses the following functions:
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 7724350d3c5f959..797a67e49c72d8b 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -50,7 +50,7 @@ Module Objects
.. c:function:: PyObject* PyModule_New(const char *name)
- Similar to :c:func:`PyImport_NewObject`, but the name is a UTF-8 encoded
+ Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded
string instead of a Unicode object.
@@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel
.. c:member:: traverseproc m_traverse
A traversal function to call during GC traversal of the module object, or
- *NULL* if not needed.
+ *NULL* if not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
.. c:member:: inquiry m_clear
A clear function to call during GC clearing of the module object, or
- *NULL* if not needed.
+ *NULL* if not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
.. c:member:: freefunc m_free
A function to call during deallocation of the module object, or *NULL* if
- not needed.
+ not needed. This function may be called before module state
+ is allocated (:c:func:`PyModule_GetState()` may return `NULL`),
+ and before the :c:member:`Py_mod_exec` function is executed.
Single-phase initialization
...........................
diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst
index 3c7605a67fa2260..296b21c132b79a4 100644
--- a/Doc/c-api/number.rst
+++ b/Doc/c-api/number.rst
@@ -280,3 +280,4 @@ Number Protocol
Returns ``1`` if *o* is an index integer (has the nb_index slot of the
tp_as_number structure filled in), and ``0`` otherwise.
+ This function always succeeds.
diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst
index e7f4fde00256db7..9ad7c571c4d8c1a 100644
--- a/Doc/c-api/objbuffer.rst
+++ b/Doc/c-api/objbuffer.rst
@@ -39,7 +39,11 @@ an object, and :c:func:`PyBuffer_Release` when the buffer view can be released.
.. c:function:: int PyObject_CheckReadBuffer(PyObject *o)
Returns ``1`` if *o* supports the single-segment readable buffer interface.
- Otherwise returns ``0``.
+ Otherwise returns ``0``. This function always succeeds.
+
+ Note that this function tries to get and release a buffer, and exceptions
+ which occur while calling correspoding functions will get suppressed.
+ To get error reporting use :c:func:`PyObject_GetBuffer()` instead.
.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len)
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index b761c808fcb7f6c..ea0aa8df31e6096 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -33,6 +33,10 @@ Object Protocol
is equivalent to the Python expression ``hasattr(o, attr_name)``. This function
always succeeds.
+ Note that exceptions which occur while calling :meth:`__getattr__` and
+ :meth:`__getattribute__` methods will get suppressed.
+ To get error reporting use :c:func:`PyObject_GetAttr()` instead.
+
.. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name)
@@ -40,6 +44,11 @@ Object Protocol
is equivalent to the Python expression ``hasattr(o, attr_name)``. This function
always succeeds.
+ Note that exceptions which occur while calling :meth:`__getattr__` and
+ :meth:`__getattribute__` methods and creating a temporary string object
+ will get suppressed.
+ To get error reporting use :c:func:`PyObject_GetAttrString()` instead.
+
.. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name)
@@ -360,8 +369,8 @@ Object Protocol
parameters must be non-*NULL*.
-.. c:function:: Py_ssize_t PyObject_Length(PyObject *o)
- Py_ssize_t PyObject_Size(PyObject *o)
+.. c:function:: Py_ssize_t PyObject_Size(PyObject *o)
+ Py_ssize_t PyObject_Length(PyObject *o)
.. index:: builtin: len
@@ -395,8 +404,8 @@ Object Protocol
.. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key)
- Delete the mapping for *key* from *o*. Returns ``-1`` on failure. This is the
- equivalent of the Python statement ``del o[key]``.
+ Remove the mapping for the object *key* from the object *o*. Return ``-1``
+ on failure. This is equivalent to the Python statement ``del o[key]``.
.. c:function:: PyObject* PyObject_Dir(PyObject *o)
diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index f1825f079be4746..6d22f35e22b1f23 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -9,7 +9,10 @@ Sequence Protocol
.. c:function:: int PySequence_Check(PyObject *o)
Return ``1`` if the object provides sequence protocol, and ``0`` otherwise.
- This function always succeeds.
+ Note that it returns ``1`` for Python classes with a :meth:`__getitem__`
+ method unless they are :class:`dict` subclasses since in general case it
+ is impossible to determine what the type of keys it supports. This
+ function always succeeds.
.. c:function:: Py_ssize_t PySequence_Size(PyObject *o)
@@ -17,9 +20,8 @@ Sequence Protocol
.. index:: builtin: len
- Returns the number of objects in sequence *o* on success, and ``-1`` on failure.
- For objects that do not provide sequence protocol, this is equivalent to the
- Python expression ``len(o)``.
+ Returns the number of objects in sequence *o* on success, and ``-1`` on
+ failure. This is equivalent to the Python expression ``len(o)``.
.. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2)
@@ -120,18 +122,27 @@ Sequence Protocol
.. index:: builtin: tuple
- Return a tuple object with the same contents as the arbitrary sequence *o* or
- *NULL* on failure. If *o* is a tuple, a new reference will be returned,
+ Return a tuple object with the same contents as the sequence or iterable *o*,
+ or *NULL* on failure. If *o* is a tuple, a new reference will be returned,
otherwise a tuple will be constructed with the appropriate contents. This is
equivalent to the Python expression ``tuple(o)``.
.. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m)
- Return the sequence *o* as a list, unless it is already a tuple or list, in
+ Return the sequence or iterable *o* as a list, unless it is already a tuple or list, in
which case *o* is returned. Use :c:func:`PySequence_Fast_GET_ITEM` to access
the members of the result. Returns *NULL* on failure. If the object is not
- a sequence, raises :exc:`TypeError` with *m* as the message text.
+ a sequence or iterable, raises :exc:`TypeError` with *m* as the message text.
+
+
+.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
+
+ Returns the length of *o*, assuming that *o* was returned by
+ :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be
+ gotten by calling :c:func:`PySequence_Size` on *o*, but
+ :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
+ or tuple.
.. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i)
@@ -156,12 +167,3 @@ Sequence Protocol
:c:func:`PySequence_GetItem` but without checking that
:c:func:`PySequence_Check` on *o* is true and without adjustment for negative
indices.
-
-
-.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o)
-
- Returns the length of *o*, assuming that *o* was returned by
- :c:func:`PySequence_Fast` and that *o* is not *NULL*. The size can also be
- gotten by calling :c:func:`PySequence_Size` on *o*, but
- :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a list
- or tuple.
diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst
index a825164918f446c..8b695e065aeffd9 100644
--- a/Doc/c-api/slice.rst
+++ b/Doc/c-api/slice.rst
@@ -56,3 +56,14 @@ Slice Objects
.. versionchanged:: 3.2
The parameter type for the *slice* parameter was ``PySliceObject*``
before.
+
+
+Ellipsis Object
+---------------
+
+
+.. c:var:: PyObject *Py_Ellipsis
+
+ The Python ``Ellipsis`` object. This object has no methods. It needs to be
+ treated just like any other object with respect to reference counts. Like
+ :c:data:`Py_None` it is a singleton object.
diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index f48119391f2bbd9..c493f6469d68e69 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -241,7 +241,7 @@ definition with the same method name.
+==================+=============+===============================+
| :attr:`name` | char \* | name of the member |
+------------------+-------------+-------------------------------+
- | :attr:`type` | int | the type of the member in the |
+ | :attr:`!type` | int | the type of the member in the |
| | | C struct |
+------------------+-------------+-------------------------------+
| :attr:`offset` | Py_ssize_t | the offset in bytes that the |
@@ -256,7 +256,7 @@ definition with the same method name.
| | | docstring |
+------------------+-------------+-------------------------------+
- :attr:`type` can be one of many ``T_`` macros corresponding to various C
+ :attr:`!type` can be one of many ``T_`` macros corresponding to various C
types. When the member is accessed in Python, it will be converted to the
equivalent Python type.
@@ -292,5 +292,46 @@ definition with the same method name.
:attr:`flags` can be ``0`` for write and read access or :c:macro:`READONLY` for
read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies
- :c:macro:`READONLY`. Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
+ :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8.
+ Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX`
members can be deleted. (They are set to *NULL*).
+
+
+.. c:type:: PyGetSetDef
+
+ Structure to define property-like access for a type. See also description of
+ the :c:member:`PyTypeObject.tp_getset` slot.
+
+ +-------------+------------------+-----------------------------------+
+ | Field | C Type | Meaning |
+ +=============+==================+===================================+
+ | name | char \* | attribute name |
+ +-------------+------------------+-----------------------------------+
+ | get | getter | C Function to get the attribute |
+ +-------------+------------------+-----------------------------------+
+ | set | setter | optional C function to set or |
+ | | | delete the attribute, if omitted |
+ | | | the attribute is readonly |
+ +-------------+------------------+-----------------------------------+
+ | doc | char \* | optional docstring |
+ +-------------+------------------+-----------------------------------+
+ | closure | void \* | optional function pointer, |
+ | | | providing additional data for |
+ | | | getter and setter |
+ +-------------+------------------+-----------------------------------+
+
+ The ``get`` function takes one :c:type:`PyObject\*` parameter (the
+ instance) and a function pointer (the associated ``closure``)::
+
+ typedef PyObject *(*getter)(PyObject *, void *);
+
+ It should return a new reference on success or *NULL* with a set exception
+ on failure.
+
+ ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and
+ the value to be set) and a function pointer (the associated ``closure``)::
+
+ typedef int (*setter)(PyObject *, PyObject *, void *);
+
+ In case the attribute should be deleted the second parameter is *NULL*.
+ Should return ``0`` on success or ``-1`` with a set exception on failure.
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index 035cdc16824886e..48e2b2bb5e91a6f 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -66,9 +66,18 @@ Operating System Utilities
surrogate character, escape the bytes using the surrogateescape error
handler instead of decoding them.
+ Encoding, highest priority to lowest priority:
+
+ * ``UTF-8`` on macOS and Android;
+ * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
+ ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
+ and :c:func:`mbstowcs` and :c:func:`wcstombs` functions use the
+ ``ISO-8859-1`` encoding.
+ * the current locale encoding (``LC_CTYPE`` locale).
+
Return a pointer to a newly allocated wide character string, use
:c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
- the number of wide characters excluding the null character into ``*size``
+ the number of wide characters excluding the null character into ``*size``.
Return ``NULL`` on decoding error or memory allocation error. If *size* is
not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
@@ -94,6 +103,15 @@ Operating System Utilities
:ref:`surrogateescape error handler `: surrogate characters
in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF.
+ Encoding, highest priority to lowest priority:
+
+ * ``UTF-8`` on macOS and Android;
+ * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
+ ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
+ and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the
+ ``ISO-8859-1`` encoding.
+ * the current locale encoding.
+
Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
to free the memory. Return ``NULL`` on encoding error or memory allocation
error
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 3922d50f80a2d21..ab8ee9abf4c46a3 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -209,7 +209,7 @@ type.
This function "steals" a reference to *o*.
-.. c:function:: PyObject* PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o)
+.. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o)
Macro equivalent of :c:func:`PyStructSequence_SetItem`.
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 60c5e73960b3a1f..4dfd53fb9f076fc 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -35,7 +35,7 @@ Type Objects
Clear the internal lookup cache. Return the current version tag.
-.. c:function:: long PyType_GetFlags(PyTypeObject* type)
+.. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type)
Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily
meant for use with `Py_LIMITED_API`; the individual flag bits are
@@ -44,6 +44,9 @@ Type Objects
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ The return type is now ``unsigned long`` rather than ``long``.
+
.. c:function:: void PyType_Modified(PyTypeObject *type)
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index ac6fd0b53fcd6d8..76515fd2c430e35 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -719,21 +719,6 @@ type objects) *must* have the :attr:`ob_size` field.
This field is not inherited by subtypes (computed attributes are inherited
through a different mechanism).
- .. XXX belongs elsewhere
-
- Docs for PyGetSetDef::
-
- typedef PyObject *(*getter)(PyObject *, void *);
- typedef int (*setter)(PyObject *, PyObject *, void *);
-
- typedef struct PyGetSetDef {
- char *name; /* attribute name */
- getter get; /* C function to get the attribute */
- setter set; /* C function to set or delete the attribute */
- char *doc; /* optional doc string */
- void *closure; /* optional additional data for getter and setter */
- } PyGetSetDef;
-
.. c:member:: PyTypeObject* PyTypeObject.tp_base
@@ -1166,21 +1151,24 @@ Mapping Object Structures
.. c:member:: lenfunc PyMappingMethods.mp_length
- This function is used by :c:func:`PyMapping_Length` and
+ This function is used by :c:func:`PyMapping_Size` and
:c:func:`PyObject_Size`, and has the same signature. This slot may be set to
*NULL* if the object has no defined length.
.. c:member:: binaryfunc PyMappingMethods.mp_subscript
- This function is used by :c:func:`PyObject_GetItem` and has the same
- signature. This slot must be filled for the :c:func:`PyMapping_Check`
- function to return ``1``, it can be *NULL* otherwise.
+ This function is used by :c:func:`PyObject_GetItem` and
+ :c:func:`PySequence_GetSlice`, and has the same signature as
+ :c:func:`!PyObject_GetItem`. This slot must be filled for the
+ :c:func:`PyMapping_Check` function to return ``1``, it can be *NULL*
+ otherwise.
.. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript
- This function is used by :c:func:`PyObject_SetItem` and
- :c:func:`PyObject_DelItem`. It has the same signature as
- :c:func:`PyObject_SetItem`, but *v* can also be set to *NULL* to delete
+ This function is used by :c:func:`PyObject_SetItem`,
+ :c:func:`PyObject_DelItem`, :c:func:`PyObject_SetSlice` and
+ :c:func:`PyObject_DelSlice`. It has the same signature as
+ :c:func:`!PyObject_SetItem`, but *v* can also be set to *NULL* to delete
an item. If this slot is *NULL*, the object does not support item
assignment and deletion.
@@ -1200,26 +1188,29 @@ Sequence Object Structures
.. c:member:: lenfunc PySequenceMethods.sq_length
- This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`,
- and has the same signature.
+ This function is used by :c:func:`PySequence_Size` and
+ :c:func:`PyObject_Size`, and has the same signature. It is also used for
+ handling negative indices via the :c:member:`~PySequenceMethods.sq_item`
+ and the :c:member:`~PySequenceMethods.sq_ass_item` slots.
.. c:member:: binaryfunc PySequenceMethods.sq_concat
This function is used by :c:func:`PySequence_Concat` and has the same
signature. It is also used by the ``+`` operator, after trying the numeric
- addition via the :c:member:`~PyTypeObject.tp_as_number.nb_add` slot.
+ addition via the :c:member:`~PyNumberMethods.nb_add` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_repeat
This function is used by :c:func:`PySequence_Repeat` and has the same
signature. It is also used by the ``*`` operator, after trying numeric
- multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_multiply`
- slot.
+ multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_item
This function is used by :c:func:`PySequence_GetItem` and has the same
- signature. This slot must be filled for the :c:func:`PySequence_Check`
+ signature. It is also used by :c:func:`PyObject_GetItem`, after trying
+ the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot.
+ This slot must be filled for the :c:func:`PySequence_Check`
function to return ``1``, it can be *NULL* otherwise.
Negative indexes are handled as follows: if the :attr:`sq_length` slot is
@@ -1230,28 +1221,36 @@ Sequence Object Structures
.. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item
This function is used by :c:func:`PySequence_SetItem` and has the same
- signature. This slot may be left to *NULL* if the object does not support
+ signature. It is also used by :c:func:`PyObject_SetItem` and
+ :c:func:`PyObject_DelItem`, after trying the item assignment and deletion
+ via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot.
+ This slot may be left to *NULL* if the object does not support
item assignment and deletion.
.. c:member:: objobjproc PySequenceMethods.sq_contains
This function may be used by :c:func:`PySequence_Contains` and has the same
signature. This slot may be left to *NULL*, in this case
- :c:func:`PySequence_Contains` simply traverses the sequence until it finds a
- match.
+ :c:func:`!PySequence_Contains` simply traverses the sequence until it
+ finds a match.
.. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat
This function is used by :c:func:`PySequence_InPlaceConcat` and has the same
- signature. It should modify its first operand, and return it.
+ signature. It should modify its first operand, and return it. This slot
+ may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceConcat`
+ will fall back to :c:func:`PySequence_Concat`. It is also used by the
+ augmented assignment ``+=``, after trying numeric inplace addition
+ via the :c:member:`~PyNumberMethods.nb_inplace_add` slot.
.. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat
This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same
- signature. It should modify its first operand, and return it.
-
-.. XXX need to explain precedence between mapping and sequence
-.. XXX explains when to implement the sq_inplace_* slots
+ signature. It should modify its first operand, and return it. This slot
+ may be left to *NULL*, in this case :c:func:`!PySequence_InPlaceRepeat`
+ will fall back to :c:func:`PySequence_Repeat`. It is also used by the
+ augmented assignment ``*=``, after trying numeric inplace multiplication
+ via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot.
.. _buffer-structs:
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 02f7ada7be7ee4e..55d5b070ff3abbd 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -773,6 +773,12 @@ system.
.. versionadded:: 3.3
+ .. versionchanged:: 3.6.5
+ The function now also uses the current locale encoding for the
+ ``surrogateescape`` error handler. Previously, :c:func:`Py_DecodeLocale`
+ was used for the ``surrogateescape``, and the current locale encoding was
+ used for ``strict``.
+
.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors)
@@ -800,6 +806,12 @@ system.
.. versionadded:: 3.3
+ .. versionchanged:: 3.6.5
+ The function now also uses the current locale encoding for the
+ ``surrogateescape`` error handler. Previously, :c:func:`Py_EncodeLocale`
+ was used for the ``surrogateescape``, and the current locale encoding was
+ used for ``strict``.
+
File System Encoding
""""""""""""""""""""
@@ -913,7 +925,7 @@ wchar_t Support
Return *NULL* on failure.
-.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode, wchar_t *w, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size)
Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most
*size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing
@@ -1312,7 +1324,7 @@ These are the "Raw Unicode Escape" codec APIs:
.. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, \
- Py_ssize_t size, const char *errors)
+ Py_ssize_t size)
Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape
and return a bytes object. Return *NULL* if an exception was raised by the codec.
@@ -1393,77 +1405,78 @@ Character Map Codecs
This codec is special in that it can be used to implement many different codecs
(and this is in fact what was done to obtain most of the standard codecs
included in the :mod:`encodings` package). The codec uses mapping to encode and
-decode characters.
-
-Decoding mappings must map single string characters to single Unicode
-characters, integers (which are then interpreted as Unicode ordinals) or ``None``
-(meaning "undefined mapping" and causing an error).
-
-Encoding mappings must map single Unicode characters to single string
-characters, integers (which are then interpreted as Latin-1 ordinals) or ``None``
-(meaning "undefined mapping" and causing an error).
-
-The mapping objects provided must only support the __getitem__ mapping
-interface.
-
-If a character lookup fails with a LookupError, the character is copied as-is
-meaning that its ordinal value will be interpreted as Unicode or Latin-1 ordinal
-resp. Because of this, mappings only need to contain those mappings which map
-characters to different code points.
+decode characters. The mapping objects provided must support the
+:meth:`__getitem__` mapping interface; dictionaries and sequences work well.
These are the mapping codec APIs:
-.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *s, Py_ssize_t size, \
+.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \
PyObject *mapping, const char *errors)
- Create a Unicode object by decoding *size* bytes of the encoded string *s* using
- the given *mapping* object. Return *NULL* if an exception was raised by the
- codec. If *mapping* is *NULL* latin-1 decoding will be done. Else it can be a
- dictionary mapping byte or a unicode string, which is treated as a lookup table.
- Byte values greater that the length of the string and U+FFFE "characters" are
- treated as "undefined mapping".
+ Create a Unicode object by decoding *size* bytes of the encoded string *s*
+ using the given *mapping* object. Return *NULL* if an exception was raised
+ by the codec.
+
+ If *mapping* is *NULL*, Latin-1 decoding will be applied. Else
+ *mapping* must map bytes ordinals (integers in the range from 0 to 255)
+ to Unicode strings, integers (which are then interpreted as Unicode
+ ordinals) or ``None``. Unmapped data bytes -- ones which cause a
+ :exc:`LookupError`, as well as ones which get mapped to ``None``,
+ ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause
+ an error.
.. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping)
- Encode a Unicode object using the given *mapping* object and return the result
- as Python string object. Error handling is "strict". Return *NULL* if an
+ Encode a Unicode object using the given *mapping* object and return the
+ result as a bytes object. Error handling is "strict". Return *NULL* if an
exception was raised by the codec.
-The following codec API is special in that maps Unicode to Unicode.
+ The *mapping* object must map Unicode ordinal integers to bytes objects,
+ integers in the range from 0 to 255 or ``None``. Unmapped character
+ ordinals (ones which cause a :exc:`LookupError`) as well as mapped to
+ ``None`` are treated as "undefined mapping" and cause an error.
-.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \
- PyObject *table, const char *errors)
-
- Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a
- character mapping *table* to it and return the resulting Unicode object. Return
- *NULL* when an exception was raised by the codec.
-
- The *mapping* table must map Unicode ordinal integers to Unicode ordinal
- integers or ``None`` (causing deletion of the character).
+.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
- Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries
- and sequences work well. Unmapped character ordinals (ones which cause a
- :exc:`LookupError`) are left untouched and are copied as-is.
+ Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
+ *mapping* object and return the result as a bytes object. Return *NULL* if
+ an exception was raised by the codec.
.. deprecated-removed:: 3.3 4.0
Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
- :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API
- `
+ :c:func:`PyUnicode_AsCharmapString` or
+ :c:func:`PyUnicode_AsEncodedString`.
-.. c:function:: PyObject* PyUnicode_EncodeCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+The following codec API is special in that maps Unicode to Unicode.
+
+.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \
PyObject *mapping, const char *errors)
- Encode the :c:type:`Py_UNICODE` buffer of the given *size* using the given
- *mapping* object and return a Python string object. Return *NULL* if an
- exception was raised by the codec.
+ Translate a Unicode object using the given *mapping* object and return the
+ resulting Unicode object. Return *NULL* if an exception was raised by the
+ codec.
+
+ The *mapping* object must map Unicode ordinal integers to Unicode strings,
+ integers (which are then interpreted as Unicode ordinals) or ``None``
+ (causing deletion of the character). Unmapped character ordinals (ones
+ which cause a :exc:`LookupError`) are left untouched and are copied as-is.
+
+
+.. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \
+ PyObject *mapping, const char *errors)
+
+ Translate a :c:type:`Py_UNICODE` buffer of the given *size* by applying a
+ character *mapping* table to it and return the resulting Unicode object.
+ Return *NULL* when an exception was raised by the codec.
.. deprecated-removed:: 3.3 4.0
Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
- :c:func:`PyUnicode_AsCharmapString` or
- :c:func:`PyUnicode_AsEncodedString`.
+ :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API
+ `
MBCS codecs for Windows
@@ -1480,8 +1493,8 @@ the user settings on the machine running the codec.
Return *NULL* if an exception was raised by the codec.
-.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, \
- const char *errors, int *consumed)
+.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \
+ const char *errors, Py_ssize_t *consumed)
If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If
*consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 6ab5942929fccea..5352938030bb7f2 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -141,7 +141,8 @@ the same library that the Python runtime is using.
Read and execute statements from a file associated with an interactive device
until EOF is reached. The user will be prompted using ``sys.ps1`` and
``sys.ps2``. *filename* is decoded from the filesystem encoding
- (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF.
+ (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative
+ number upon failure.
.. c:var:: int (*PyOS_InputHook)(void)
@@ -305,7 +306,7 @@ the same library that the Python runtime is using.
Evaluate a precompiled code object, given a particular environment for its
evaluation. This environment consists of a dictionary of global variables,
a mapping object of local variables, arrays of arguments, keywords and
- defaults, a dictionary of default values for :ref:`keyword-only\
+ defaults, a dictionary of default values for :ref:`keyword-only
` arguments and a closure tuple of cells.
diff --git a/Doc/conf.py b/Doc/conf.py
index b1bb6208bb4b8e6..fab963e23a0fee0 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -13,7 +13,7 @@
# ---------------------
extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest',
- 'pyspecific', 'c_annotations']
+ 'pyspecific', 'c_annotations', 'escape4chm']
# General substitutions.
project = 'Python'
@@ -37,8 +37,11 @@
needs_sphinx = '1.2'
# Ignore any .rst files in the venv/ directory.
-exclude_patterns = ['venv/*']
+venvdir = os.getenv('VENVDIR', 'venv')
+exclude_patterns = [venvdir+'/*', 'README.rst']
+# Avoid a warning with Sphinx >= 2.0
+master_doc = 'contents'
# Options for HTML output
# -----------------------
@@ -88,11 +91,27 @@
# Options for LaTeX output
# ------------------------
+latex_engine = 'xelatex'
+
+# Get LaTeX to handle Unicode correctly
+latex_elements = {
+}
+
+# Additional stuff for the LaTeX preamble.
+latex_elements['preamble'] = r'''
+\authoraddress{
+ \sphinxstrong{Python Software Foundation}\\
+ Email: \sphinxemail{docs@python.org}
+}
+\let\Verbatim=\OriginalVerbatim
+\let\endVerbatim=\endOriginalVerbatim
+'''
+
# The paper size ('letter' or 'a4').
-latex_paper_size = 'a4'
+latex_elements['papersize'] = 'a4'
# The font size ('10pt', '11pt' or '12pt').
-latex_font_size = '10pt'
+latex_elements['pointsize'] = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
@@ -125,22 +144,9 @@
for fn in os.listdir('howto')
if fn.endswith('.rst') and fn != 'index.rst')
-# Additional stuff for the LaTeX preamble.
-latex_preamble = r'''
-\authoraddress{
- \strong{Python Software Foundation}\\
- Email: \email{docs@python.org}
-}
-\let\Verbatim=\OriginalVerbatim
-\let\endVerbatim=\endOriginalVerbatim
-'''
-
# Documents to append as an appendix to all manuals.
latex_appendices = ['glossary', 'about', 'license', 'copyright']
-# Get LaTeX to handle Unicode correctly
-latex_elements = {'inputenc': r'\usepackage[utf8x]{inputenc}', 'utf8extra': ''}
-
# Options for Epub output
# -----------------------
diff --git a/Doc/copyright.rst b/Doc/copyright.rst
index 22d7705846ea936..4191c0bb63a2c1f 100644
--- a/Doc/copyright.rst
+++ b/Doc/copyright.rst
@@ -4,7 +4,7 @@ Copyright
Python and this documentation is:
-Copyright © 2001-2016 Python Software Foundation. All rights reserved.
+Copyright © 2001-2021 Python Software Foundation. All rights reserved.
Copyright © 2000 BeOpen.com. All rights reserved.
diff --git a/Doc/data/python3.6m.abi b/Doc/data/python3.6m.abi
new file mode 100644
index 000000000000000..804f8aa2e3160eb
--- /dev/null
+++ b/Doc/data/python3.6m.abi
@@ -0,0 +1,14626 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 8b469f4aac9a489..0ac62abc7d4cd9c 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -516,6 +516,9 @@ Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3
PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules
PyImport_AddModule:const char*:name::
+PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules
+PyImport_AddModuleObject:PyObject*:name:0:
+
PyImport_Cleanup:void:::
PyImport_ExecCodeModule:PyObject*::+1:
@@ -527,6 +530,21 @@ PyImport_ExecCodeModuleEx:const char*:name::
PyImport_ExecCodeModuleEx:PyObject*:co:0:
PyImport_ExecCodeModuleEx:const char*:pathname::
+PyImport_ExecCodeModuleObject:PyObject*::+1:
+PyImport_ExecCodeModuleObject:const char*:name::
+PyImport_ExecCodeModuleObject:PyObject*:co:0:
+PyImport_ExecCodeModuleObject:PyObject*:pathname:0:
+PyImport_ExecCodeModuleObject:PyObject*:cpathname:0:
+
+PyImport_ExecCodeModuleWithPathnames:PyObject*::+1:
+PyImport_ExecCodeModuleWithPathnames:const char*:name::
+PyImport_ExecCodeModuleWithPathnames:PyObject*:co:0:
+PyImport_ExecCodeModuleWithPathnames:const char*:pathname::
+PyImport_ExecCodeModuleWithPathnames:const char*:cpathname::
+
+PyImport_GetImporter:PyObject*::+1:
+PyImport_GetImporter:PyObject*:path:0:
+
PyImport_GetMagicNumber:long:::
PyImport_GetModuleDict:PyObject*::0:
@@ -537,6 +555,9 @@ PyImport_Import:PyObject*:name:0:
PyImport_ImportFrozenModule:int:::
PyImport_ImportFrozenModule:const char*:::
+PyImport_ImportFrozenModuleObject:int:::
+PyImport_ImportFrozenModuleObject:PyObject*::+1:
+
PyImport_ImportModule:PyObject*::+1:
PyImport_ImportModule:const char*:name::
@@ -553,6 +574,13 @@ PyImport_ImportModuleLevel:PyObject*:locals:0:???
PyImport_ImportModuleLevel:PyObject*:fromlist:0:???
PyImport_ImportModuleLevel:int:level::
+PyImport_ImportModuleLevelObject:PyObject*::+1:
+PyImport_ImportModuleLevelObject:PyObject*:name:0:
+PyImport_ImportModuleLevelObject:PyObject*:globals:0:???
+PyImport_ImportModuleLevelObject:PyObject*:locals:0:???
+PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:???
+PyImport_ImportModuleLevelObject:int:level::
+
PyImport_ReloadModule:PyObject*::+1:
PyImport_ReloadModule:PyObject*:m:0:
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
index 82ecd2c1ef45fbe..b0dccb3db1ddf4b 100644
--- a/Doc/distributing/index.rst
+++ b/Doc/distributing/index.rst
@@ -31,7 +31,7 @@ installing other Python projects, refer to the
Key terms
=========
-* the `Python Packaging Index `__ is a public
+* the `Python Packaging Index `__ is a public
repository of open source licensed packages made available for use by
other Python users
* the `Python Packaging Authority
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index b5b7731074c2ca3..207f43864c93189 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -78,7 +78,7 @@ setup script). Indirectly provides the :class:`distutils.dist.Distribution` and
| | be built | :class:`distutils.core.Extension` |
+--------------------+--------------------------------+-------------------------------------------------------------+
| *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI |
- | | package | `_. |
+ | | package | `_. |
+--------------------+--------------------------------+-------------------------------------------------------------+
| *distclass* | the :class:`Distribution` | a subclass of |
| | class to use | :class:`distutils.core.Distribution` |
@@ -183,8 +183,9 @@ the full reference.
| *sources* | list of source filenames, | a list of strings |
| | relative to the distribution | |
| | root (where the setup script | |
- | | lives), in Unix form (slash- | |
- | | separated) for portability. | |
+ | | lives), in Unix form | |
+ | | (slash-separated) for | |
+ | | portability. | |
| | Source files may be C, C++, | |
| | SWIG (.i), platform-specific | |
| | resource files, or whatever | |
@@ -936,7 +937,7 @@ timestamp dependency analysis.
.. function:: newer_group(sources, target[, missing='error'])
Return true if *target* is out-of-date with respect to any file listed in
- *sources* In other words, if *target* exists and is newer than every file in
+ *sources*. In other words, if *target* exists and is newer than every file in
*sources*, return false; otherwise return true. *missing* controls what we do
when a source file is missing; the default (``'error'``) is to blow up with an
:exc:`OSError` from inside :func:`os.stat`; if it is ``'ignore'``, we silently
@@ -1565,8 +1566,8 @@ lines, and joining lines with backslashes.
+------------------+--------------------------------+---------+
| option name | description | default |
+==================+================================+=========+
- | *strip_comments* | strip from ``'#'`` to end-of- | true |
- | | line, as well as any | |
+ | *strip_comments* | strip from ``'#'`` to | true |
+ | | end-of-line, as well as any | |
| | whitespace leading up to the | |
| | ``'#'``\ ---unless it is | |
| | escaped by a backslash | |
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index bbd2a8ce831cee1..92d796fb7472112 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -21,7 +21,7 @@ specialty---writing code and creating source distributions---while an
intermediary species called *packagers* springs up to turn source distributions
into built distributions for as many platforms as there are packagers.
-Of course, the module developer could be his own packager; or the packager could
+Of course, the module developer could be their own packager; or the packager could
be a volunteer "out there" somewhere who has access to a platform which the
original developer does not; or it could be software periodically grabbing new
source distributions and turning them into built distributions for as many
@@ -63,9 +63,9 @@ distribution to generate: for example, ::
python setup.py bdist --format=zip
-would, when run on a Unix system, create :file:`Distutils-1.0.{plat}.zip`\
----again, this archive would be unpacked from the root directory to install the
-Distutils.
+would, when run on a Unix system, create
+:file:`Distutils-1.0.{plat}.zip`\ ---again, this archive would be unpacked
+from the root directory to install the Distutils.
The available formats for built distributions are:
diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst
index 21f1acdace5f194..0874d05fe703a17 100644
--- a/Doc/distutils/configfile.rst
+++ b/Doc/distutils/configfile.rst
@@ -13,8 +13,8 @@ edit is a cheap and easy way to solicit it. Configuration files also let you
provide default values for any command option, which the installer can then
override either on the command-line or by editing the config file.
-The setup configuration file is a useful middle-ground between the setup script
----which, ideally, would be opaque to installers [#]_---and the command-line to
+The setup configuration file is a useful middle-ground between the setup
+script---which, ideally, would be opaque to installers [#]_---and the command-line to
the setup script, which is outside of your control and entirely up to the
installer. In fact, :file:`setup.cfg` (and any other Distutils configuration
files present on the target system) are processed after the contents of the
@@ -36,7 +36,9 @@ consequences:
* installers can override anything in :file:`setup.cfg` using the command-line
options to :file:`setup.py`
-The basic syntax of the configuration file is simple::
+The basic syntax of the configuration file is simple:
+
+.. code-block:: ini
[command]
option=value
@@ -51,9 +53,11 @@ option values can be split across multiple lines simply by indenting the
continuation lines.
You can find out the list of options supported by a particular command with the
-universal :option:`!--help` option, e.g. ::
+universal :option:`!--help` option, e.g.
+
+.. code-block:: shell-session
- > python setup.py --help build_ext
+ $ python setup.py --help build_ext
[...]
Options for 'build_ext' command:
--build-lib (-b) directory for compiled extension modules
@@ -75,14 +79,18 @@ For example, say you want your extensions to be built "in-place"---that is, you
have an extension :mod:`pkg.ext`, and you want the compiled extension file
(:file:`ext.so` on Unix, say) to be put in the same source directory as your
pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the
-:option:`!--inplace` option on the command-line to ensure this::
+:option:`!--inplace` option on the command-line to ensure this:
+
+.. code-block:: sh
python setup.py build_ext --inplace
But this requires that you always specify the :command:`build_ext` command
explicitly, and remember to provide :option:`!--inplace`. An easier way is to
"set and forget" this option, by encoding it in :file:`setup.cfg`, the
-configuration file for this distribution::
+configuration file for this distribution:
+
+.. code-block:: ini
[build_ext]
inplace=1
@@ -103,7 +111,9 @@ information comes from the setup script, and some is automatically generated by
the Distutils (such as the list of files installed). But some of it has to be
supplied as options to :command:`bdist_rpm`, which would be very tedious to do
on the command-line for every run. Hence, here is a snippet from the Distutils'
-own :file:`setup.cfg`::
+own :file:`setup.cfg`:
+
+.. code-block:: ini
[bdist_rpm]
release = 1
diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst
index 1f5be9cdb29a974..4e2761d8a7d0460 100644
--- a/Doc/distutils/examples.rst
+++ b/Doc/distutils/examples.rst
@@ -321,7 +321,7 @@ You can read back this static file, by using the
>>> metadata.description
'Easily download, build, install, upgrade, and uninstall Python packages'
-Notice that the class can also be instanciated with a metadata file path to
+Notice that the class can also be instantiated with a metadata file path to
loads its values::
>>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info'
diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst
index 8f46bd74c5b01a1..7721484fe737178 100644
--- a/Doc/distutils/introduction.rst
+++ b/Doc/distutils/introduction.rst
@@ -94,7 +94,7 @@ containing your setup script :file:`setup.py`, and your module :file:`foo.py`.
The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and
will unpack into a directory :file:`foo-1.0`.
-If an end-user wishes to install your :mod:`foo` module, all she has to do is
+If an end-user wishes to install your :mod:`foo` module, all they have to do is
download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the
:file:`foo-1.0` directory---run ::
@@ -193,8 +193,8 @@ modules using the Distutils:
module distribution
a collection of Python modules distributed together as a single downloadable
resource and meant to be installed *en masse*. Examples of some well-known
- module distributions are NumPy, SciPy, PIL (the Python Imaging
- Library), or mxBase. (This would be called a *package*, except that term is
+ module distributions are NumPy, SciPy, Pillow,
+ or mxBase. (This would be called a *package*, except that term is
already taken in the Python context: a single module distribution may contain
zero, one, or many Python packages.)
diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst
index 44556e3df9c5b1b..50cb74f2b6ca905 100644
--- a/Doc/distutils/packageindex.rst
+++ b/Doc/distutils/packageindex.rst
@@ -156,7 +156,9 @@ The :command:`register` and :command:`upload` commands both check for the
existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`.
If this file exists, the command uses the username, password, and repository
URL configured in the file. The format of a :file:`.pypirc` file is as
-follows::
+follows:
+
+.. code-block:: ini
[distutils]
index-servers =
@@ -179,7 +181,9 @@ Each section describing a repository defines three variables:
will be prompt to type it when needed.
If you want to define another server a new section can be created and
-listed in the *index-servers* variable::
+listed in the *index-servers* variable:
+
+.. code-block:: ini
[distutils]
index-servers =
@@ -246,4 +250,4 @@ without warnings does not guarantee that PyPI will convert the content
successfully.
-.. _Python Package Index (PyPI): https://pypi.python.org/pypi
+.. _Python Package Index (PyPI): https://pypi.org
diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst
index 38e0202e4ac1f6f..21d569b65c3f526 100644
--- a/Doc/distutils/setupscript.rst
+++ b/Doc/distutils/setupscript.rst
@@ -609,7 +609,7 @@ Notes:
(4)
These fields should not be used if your package is to be compatible with Python
versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website
- `_.
+ `_.
(5)
The ``long_description`` field is used by PyPI when you are
diff --git a/Doc/docutils.conf b/Doc/docutils.conf
new file mode 100644
index 000000000000000..bda4f5dc2351dcb
--- /dev/null
+++ b/Doc/docutils.conf
@@ -0,0 +1,2 @@
+[restructuredtext parser]
+smartquotes-locales: ja: ""''
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index ab2f61614afc4ec..7e4fc19db83b2a6 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -323,7 +323,7 @@ options. In this case, the :mod:`sysconfig` module is a useful tool to
programmatically extract the configuration values that you will want to
combine together. For example:
-.. code-block:: python
+.. code-block:: pycon
>>> import sysconfig
>>> sysconfig.get_config_var('LIBS')
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index 4ac820a432b9b01..2a288404fd13fd0 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -40,9 +40,11 @@ A Simple Example
Let's create an extension module called ``spam`` (the favorite food of Monty
Python fans...) and let's say we want to create a Python interface to the C
-library function :c:func:`system`. [#]_ This function takes a null-terminated
+library function :c:func:`system` [#]_. This function takes a null-terminated
character string as argument and returns an integer. We want this function to
-be callable from Python as follows::
+be callable from Python as follows:
+
+.. code-block:: pycon
>>> import spam
>>> status = spam.system("ls -l")
@@ -333,12 +335,12 @@ function.
The method table must be referenced in the module definition structure::
static struct PyModuleDef spammodule = {
- PyModuleDef_HEAD_INIT,
- "spam", /* name of module */
- spam_doc, /* module documentation, may be NULL */
- -1, /* size of per-interpreter state of the module,
- or -1 if the module keeps state in global variables. */
- SpamMethods
+ PyModuleDef_HEAD_INIT,
+ "spam", /* name of module */
+ spam_doc, /* module documentation, may be NULL */
+ -1, /* size of per-interpreter state of the module,
+ or -1 if the module keeps state in global variables. */
+ SpamMethods
};
This structure, in turn, must be passed to the interpreter in the module's
@@ -438,7 +440,9 @@ part of the Python interpreter, you will have to change the configuration setup
and rebuild the interpreter. Luckily, this is very simple on Unix: just place
your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory
of an unpacked source distribution, add a line to the file
-:file:`Modules/Setup.local` describing your file::
+:file:`Modules/Setup.local` describing your file:
+
+.. code-block:: sh
spam spammodule.o
@@ -449,7 +453,9 @@ subdirectory, but then you must first rebuild :file:`Makefile` there by running
:file:`Setup` file.)
If your module requires additional libraries to link with, these can be listed
-on the line in the configuration file as well, for instance::
+on the line in the configuration file as well, for instance:
+
+.. code-block:: sh
spam spammodule.o -lX11
@@ -538,8 +544,9 @@ or more format codes between parentheses. For example::
:c:func:`PyObject_CallObject` returns a Python object pointer: this is the return
value of the Python function. :c:func:`PyObject_CallObject` is
"reference-count-neutral" with respect to its arguments. In the example a new
-tuple was created to serve as the argument list, which is :c:func:`Py_DECREF`\
--ed immediately after the :c:func:`PyObject_CallObject` call.
+tuple was created to serve as the argument list, which is
+:c:func:`Py_DECREF`\ -ed immediately after the :c:func:`PyObject_CallObject`
+call.
The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand
new object, or it is an existing object whose reference count has been
@@ -917,7 +924,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The
borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must
not hold on to the object longer than the owner from which it was borrowed.
Using a borrowed reference after the owner has disposed of it risks using freed
-memory and should be avoided completely. [#]_
+memory and should be avoided completely [#]_.
The advantage of borrowing over owning a reference is that you don't need to
take care of disposing of the reference on all possible paths through the code
@@ -1088,7 +1095,7 @@ checking.
The C function calling mechanism guarantees that the argument list passed to C
functions (``args`` in the examples) is never *NULL* --- in fact it guarantees
-that it is always a tuple. [#]_
+that it is always a tuple [#]_.
It is a severe error to ever let a *NULL* pointer "escape" to the Python user.
diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst
index 9eec8c273af3011..533ed985f829062 100644
--- a/Doc/extending/index.rst
+++ b/Doc/extending/index.rst
@@ -26,9 +26,11 @@ Recommended third party tools
=============================
This guide only covers the basic tools for creating extensions provided
-as part of this version of CPython. Third party tools like Cython,
-``cffi``, SWIG and Numba offer both simpler and more sophisticated
-approaches to creating C and C++ extensions for Python.
+as part of this version of CPython. Third party tools like
+`Cython `_, `cffi `_,
+`SWIG `_ and `Numba `_
+offer both simpler and more sophisticated approaches to creating C and C++
+extensions for Python.
.. seealso::
@@ -52,6 +54,7 @@ C extensions.
:numbered:
extending.rst
+ newtypes_tutorial.rst
newtypes.rst
building.rst
windows.rst
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index b8ce4377877e707..b55796421c0fbce 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -1,885 +1,11 @@
.. highlightlang:: c
-
-.. _defining-new-types:
-
-******************
-Defining New Types
-******************
-
-.. sectionauthor:: Michael Hudson
-.. sectionauthor:: Dave Kuhlman
-.. sectionauthor:: Jim Fulton
-
-
-As mentioned in the last chapter, Python allows the writer of an extension
-module to define new types that can be manipulated from Python code, much like
-strings and lists in core Python.
-
-This is not hard; the code for all extension types follows a pattern, but there
-are some details that you need to understand before you can get started.
-
-
-.. _dnt-basics:
-
-The Basics
-==========
-
-The Python runtime sees all Python objects as variables of type
-:c:type:`PyObject\*`, which serves as a "base type" for all Python objects.
-:c:type:`PyObject` itself only contains the refcount and a pointer to the
-object's "type object". This is where the action is; the type object determines
-which (C) functions get called when, for instance, an attribute gets looked
-up on an object or it is multiplied by another object. These C functions
-are called "type methods".
-
-So, if you want to define a new object type, you need to create a new type
-object.
-
-This sort of thing can only be explained by example, so here's a minimal, but
-complete, module that defines a new type:
-
-.. literalinclude:: ../includes/noddy.c
-
-
-Now that's quite a bit to take in at once, but hopefully bits will seem familiar
-from the last chapter.
-
-The first bit that will be new is::
-
- typedef struct {
- PyObject_HEAD
- } noddy_NoddyObject;
-
-This is what a Noddy object will contain---in this case, nothing more than what
-every Python object contains---a field called ``ob_base`` of type
-:c:type:`PyObject`. :c:type:`PyObject` in turn, contains an ``ob_refcnt``
-field and a pointer to a type object. These can be accessed using the macros
-:c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE` respectively. These are the fields
-the :c:macro:`PyObject_HEAD` macro brings in. The reason for the macro is to
-standardize the layout and to enable special debugging fields in debug builds.
-
-Note that there is no semicolon after the :c:macro:`PyObject_HEAD` macro;
-one is included in the macro definition. Be wary of adding one by
-accident; it's easy to do from habit, and your compiler might not complain,
-but someone else's probably will! (On Windows, MSVC is known to call this an
-error and refuse to compile the code.)
-
-For contrast, let's take a look at the corresponding definition for standard
-Python floats::
-
- typedef struct {
- PyObject_HEAD
- double ob_fval;
- } PyFloatObject;
-
-Moving on, we come to the crunch --- the type object. ::
-
- static PyTypeObject noddy_NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(noddy_NoddyObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_as_async */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- "Noddy objects", /* tp_doc */
- };
-
-Now if you go and look up the definition of :c:type:`PyTypeObject` in
-:file:`object.h` you'll see that it has many more fields that the definition
-above. The remaining fields will be filled with zeros by the C compiler, and
-it's common practice to not specify them explicitly unless you need them.
-
-This is so important that we're going to pick the top of it apart still
-further::
-
- PyVarObject_HEAD_INIT(NULL, 0)
-
-This line is a bit of a wart; what we'd like to write is::
-
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
-
-as the type of a type object is "type", but this isn't strictly conforming C and
-some compilers complain. Fortunately, this member will be filled in for us by
-:c:func:`PyType_Ready`. ::
-
- "noddy.Noddy", /* tp_name */
-
-The name of our type. This will appear in the default textual representation of
-our objects and in some error messages, for example::
-
- >>> "" + noddy.new_noddy()
- Traceback (most recent call last):
- File "", line 1, in ?
- TypeError: cannot add type "noddy.Noddy" to string
-
-Note that the name is a dotted name that includes both the module name and the
-name of the type within the module. The module in this case is :mod:`noddy` and
-the type is :class:`Noddy`, so we set the type name to :class:`noddy.Noddy`.
-One side effect of using an undotted name is that the pydoc documentation tool
-will not list the new type in the module documentation. ::
-
- sizeof(noddy_NoddyObject), /* tp_basicsize */
-
-This is so that Python knows how much memory to allocate when you call
-:c:func:`PyObject_New`.
-
-.. note::
-
- If you want your type to be subclassable from Python, and your type has the same
- :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple
- inheritance. A Python subclass of your type will have to list your type first
- in its :attr:`~class.__bases__`, or else it will not be able to call your type's
- :meth:`__new__` method without getting an error. You can avoid this problem by
- ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its
- base type does. Most of the time, this will be true anyway, because either your
- base type will be :class:`object`, or else you will be adding data members to
- your base type, and therefore increasing its size.
-
-::
-
- 0, /* tp_itemsize */
-
-This has to do with variable length objects like lists and strings. Ignore this
-for now.
-
-Skipping a number of type methods that we don't provide, we set the class flags
-to :const:`Py_TPFLAGS_DEFAULT`. ::
-
- Py_TPFLAGS_DEFAULT, /* tp_flags */
-
-All types should include this constant in their flags. It enables all of the
-members defined until at least Python 3.3. If you need further members,
-you will need to OR the corresponding flags.
-
-We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
-
- "Noddy objects", /* tp_doc */
-
-Now we get into the type methods, the things that make your objects different
-from the others. We aren't going to implement any of these in this version of
-the module. We'll expand this example later to have more interesting behavior.
-
-For now, all we want to be able to do is to create new :class:`Noddy` objects.
-To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation.
-In this case, we can just use the default implementation provided by the API
-function :c:func:`PyType_GenericNew`. We'd like to just assign this to the
-:c:member:`~PyTypeObject.tp_new` slot, but we can't, for portability sake, On some platforms or
-compilers, we can't statically initialize a structure member with a function
-defined in another C module, so, instead, we'll assign the :c:member:`~PyTypeObject.tp_new` slot
-in the module initialization function just before calling
-:c:func:`PyType_Ready`::
-
- noddy_NoddyType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&noddy_NoddyType) < 0)
- return;
-
-All the other type methods are *NULL*, so we'll go over them later --- that's
-for a later section!
-
-Everything else in the file should be familiar, except for some code in
-:c:func:`PyInit_noddy`::
-
- if (PyType_Ready(&noddy_NoddyType) < 0)
- return;
-
-This initializes the :class:`Noddy` type, filing in a number of members,
-including :attr:`ob_type` that we initially set to *NULL*. ::
-
- PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType);
-
-This adds the type to the module dictionary. This allows us to create
-:class:`Noddy` instances by calling the :class:`Noddy` class::
-
- >>> import noddy
- >>> mynoddy = noddy.Noddy()
-
-That's it! All that remains is to build it; put the above code in a file called
-:file:`noddy.c` and ::
-
- from distutils.core import setup, Extension
- setup(name="noddy", version="1.0",
- ext_modules=[Extension("noddy", ["noddy.c"])])
-
-in a file called :file:`setup.py`; then typing
-
-.. code-block:: shell-session
-
- $ python setup.py build
-
-at a shell should produce a file :file:`noddy.so` in a subdirectory; move to
-that directory and fire up Python --- you should be able to ``import noddy`` and
-play around with Noddy objects.
-
-That wasn't so hard, was it?
-
-Of course, the current Noddy type is pretty uninteresting. It has no data and
-doesn't do anything. It can't even be subclassed.
-
-
-Adding data and methods to the Basic example
---------------------------------------------
-
-Let's extend the basic example to add some data and methods. Let's also make
-the type usable as a base class. We'll create a new module, :mod:`noddy2` that
-adds these capabilities:
-
-.. literalinclude:: ../includes/noddy2.c
-
-
-This version of the module has a number of changes.
-
-We've added an extra include::
-
- #include
-
-This include provides declarations that we use to handle attributes, as
-described a bit later.
-
-The name of the :class:`Noddy` object structure has been shortened to
-:class:`Noddy`. The type object name has been shortened to :class:`NoddyType`.
-
-The :class:`Noddy` type now has three data attributes, *first*, *last*, and
-*number*. The *first* and *last* variables are Python strings containing first
-and last names. The *number* attribute is an integer.
-
-The object structure is updated accordingly::
-
- typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
- } Noddy;
-
-Because we now have data to manage, we have to be more careful about object
-allocation and deallocation. At a minimum, we need a deallocation method::
-
- static void
- Noddy_dealloc(Noddy* self)
- {
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
- }
-
-which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member::
-
- (destructor)Noddy_dealloc, /*tp_dealloc*/
-
-This method decrements the reference counts of the two Python attributes. We use
-:c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members
-could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type
-to free the object's memory. Note that the object's type might not be
-:class:`NoddyType`, because the object may be an instance of a subclass.
-
-We want to make sure that the first and last names are initialized to empty
-strings, so we provide a new method::
-
- static PyObject *
- Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
- {
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
- }
-
-and install it in the :c:member:`~PyTypeObject.tp_new` member::
-
- Noddy_new, /* tp_new */
-
-The new member is responsible for creating (as opposed to initializing) objects
-of the type. It is exposed in Python as the :meth:`__new__` method. See the
-paper titled "Unifying types and classes in Python" for a detailed discussion of
-the :meth:`__new__` method. One reason to implement a new method is to assure
-the initial values of instance variables. In this case, we use the new method
-to make sure that the initial values of the members :attr:`first` and
-:attr:`last` are not *NULL*. If we didn't care whether the initial values were
-*NULL*, we could have used :c:func:`PyType_GenericNew` as our new method, as we
-did before. :c:func:`PyType_GenericNew` initializes all of the instance variable
-members to *NULL*.
-
-The new method is a static method that is passed the type being instantiated and
-any arguments passed when the type was called, and that returns the new object
-created. New methods always accept positional and keyword arguments, but they
-often ignore the arguments, leaving the argument handling to initializer
-methods. Note that if the type supports subclassing, the type passed may not be
-the type being defined. The new method calls the :c:member:`~PyTypeObject.tp_alloc` slot to
-allocate memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather
-:c:func:`PyType_Ready` fills it for us by inheriting it from our base class,
-which is :class:`object` by default. Most types use the default allocation.
-
-.. note::
-
- If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's
- :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method
- to call using method resolution order at runtime. Always statically determine
- what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via
- ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your
- type that also inherit from other Python-defined classes may not work correctly.
- (Specifically, you may not be able to create instances of such subclasses
- without getting a :exc:`TypeError`.)
-
-We provide an initialization function::
-
- static int
- Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
- {
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
- }
-
-by filling the :c:member:`~PyTypeObject.tp_init` slot. ::
-
- (initproc)Noddy_init, /* tp_init */
-
-The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It
-is used to initialize an object after it's created. Unlike the new method, we
-can't guarantee that the initializer is called. The initializer isn't called
-when unpickling objects and it can be overridden. Our initializer accepts
-arguments to provide initial values for our instance. Initializers always accept
-positional and keyword arguments. Initializers should return either 0 on
-success or -1 on error.
-
-Initializers can be called multiple times. Anyone can call the :meth:`__init__`
-method on our objects. For this reason, we have to be extra careful when
-assigning the new values. We might be tempted, for example to assign the
-:attr:`first` member like this::
-
- if (first) {
- Py_XDECREF(self->first);
- Py_INCREF(first);
- self->first = first;
- }
-
-But this would be risky. Our type doesn't restrict the type of the
-:attr:`first` member, so it could be any kind of object. It could have a
-destructor that causes code to be executed that tries to access the
-:attr:`first` member. To be paranoid and protect ourselves against this
-possibility, we almost always reassign members before decrementing their
-reference counts. When don't we have to do this?
-
-* when we absolutely know that the reference count is greater than 1
-
-* when we know that deallocation of the object [#]_ will not cause any calls
- back into our type's code
-
-* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when
- garbage-collections is not supported [#]_
-
-We want to expose our instance variables as attributes. There are a
-number of ways to do that. The simplest way is to define member definitions::
-
- static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
- };
-
-and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot::
-
- Noddy_members, /* tp_members */
-
-Each member definition has a member name, type, offset, access flags and
-documentation string. See the :ref:`Generic-Attribute-Management` section below for
-details.
-
-A disadvantage of this approach is that it doesn't provide a way to restrict the
-types of objects that can be assigned to the Python attributes. We expect the
-first and last names to be strings, but any Python objects can be assigned.
-Further, the attributes can be deleted, setting the C pointers to *NULL*. Even
-though we can make sure the members are initialized to non-*NULL* values, the
-members can be set to *NULL* if the attributes are deleted.
-
-We define a single method, :meth:`name`, that outputs the objects name as the
-concatenation of the first and last names. ::
-
- static PyObject *
- Noddy_name(Noddy* self)
- {
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
- }
-
-The method is implemented as a C function that takes a :class:`Noddy` (or
-:class:`Noddy` subclass) instance as the first argument. Methods always take an
-instance as the first argument. Methods often take positional and keyword
-arguments as well, but in this case we don't take any and don't need to accept
-a positional argument tuple or keyword argument dictionary. This method is
-equivalent to the Python method::
-
- def name(self):
- return "%s %s" % (self.first, self.last)
-
-Note that we have to check for the possibility that our :attr:`first` and
-:attr:`last` members are *NULL*. This is because they can be deleted, in which
-case they are set to *NULL*. It would be better to prevent deletion of these
-attributes and to restrict the attribute values to be strings. We'll see how to
-do that in the next section.
-
-Now that we've defined the method, we need to create an array of method
-definitions::
-
- static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
- };
-
-and assign them to the :c:member:`~PyTypeObject.tp_methods` slot::
-
- Noddy_methods, /* tp_methods */
-
-Note that we used the :const:`METH_NOARGS` flag to indicate that the method is
-passed no arguments.
-
-Finally, we'll make our type usable as a base class. We've written our methods
-carefully so far so that they don't make any assumptions about the type of the
-object being created or used, so all we need to do is to add the
-:const:`Py_TPFLAGS_BASETYPE` to our class flag definition::
-
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
-
-We rename :c:func:`PyInit_noddy` to :c:func:`PyInit_noddy2` and update the module
-name in the :c:type:`PyModuleDef` struct.
-
-Finally, we update our :file:`setup.py` file to build the new module::
-
- from distutils.core import setup, Extension
- setup(name="noddy", version="1.0",
- ext_modules=[
- Extension("noddy", ["noddy.c"]),
- Extension("noddy2", ["noddy2.c"]),
- ])
-
-
-Providing finer control over data attributes
---------------------------------------------
-
-In this section, we'll provide finer control over how the :attr:`first` and
-:attr:`last` attributes are set in the :class:`Noddy` example. In the previous
-version of our module, the instance variables :attr:`first` and :attr:`last`
-could be set to non-string values or even deleted. We want to make sure that
-these attributes always contain strings.
-
-.. literalinclude:: ../includes/noddy3.c
-
-
-To provide greater control, over the :attr:`first` and :attr:`last` attributes,
-we'll use custom getter and setter functions. Here are the functions for
-getting and setting the :attr:`first` attribute::
-
- Noddy_getfirst(Noddy *self, void *closure)
- {
- Py_INCREF(self->first);
- return self->first;
- }
-
- static int
- Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
- {
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a str");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
- }
-
-The getter function is passed a :class:`Noddy` object and a "closure", which is
-void pointer. In this case, the closure is ignored. (The closure supports an
-advanced usage in which definition data is passed to the getter and setter. This
-could, for example, be used to allow a single set of getter and setter functions
-that decide the attribute to get or set based on data in the closure.)
-
-The setter function is passed the :class:`Noddy` object, the new value, and the
-closure. The new value may be *NULL*, in which case the attribute is being
-deleted. In our setter, we raise an error if the attribute is deleted or if the
-attribute value is not a string.
-
-We create an array of :c:type:`PyGetSetDef` structures::
-
- static PyGetSetDef Noddy_getseters[] = {
- {"first",
- (getter)Noddy_getfirst, (setter)Noddy_setfirst,
- "first name",
- NULL},
- {"last",
- (getter)Noddy_getlast, (setter)Noddy_setlast,
- "last name",
- NULL},
- {NULL} /* Sentinel */
- };
-
-and register it in the :c:member:`~PyTypeObject.tp_getset` slot::
-
- Noddy_getseters, /* tp_getset */
-
-to register our attribute getters and setters.
-
-The last item in a :c:type:`PyGetSetDef` structure is the closure mentioned
-above. In this case, we aren't using the closure, so we just pass *NULL*.
-
-We also remove the member definitions for these attributes::
-
- static PyMemberDef Noddy_members[] = {
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
- };
-
-We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to
-be passed::
-
- static int
- Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
- {
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
- }
-
- return 0;
- }
-
-With these changes, we can assure that the :attr:`first` and :attr:`last`
-members are never *NULL* so we can remove checks for *NULL* values in almost all
-cases. This means that most of the :c:func:`Py_XDECREF` calls can be converted to
-:c:func:`Py_DECREF` calls. The only place we can't change these calls is in the
-deallocator, where there is the possibility that the initialization of these
-members failed in the constructor.
-
-We also rename the module initialization function and module name in the
-initialization function, as we did before, and we add an extra definition to the
-:file:`setup.py` file.
-
-
-Supporting cyclic garbage collection
-------------------------------------
-
-Python has a cyclic-garbage collector that can identify unneeded objects even
-when their reference counts are not zero. This can happen when objects are
-involved in cycles. For example, consider::
-
- >>> l = []
- >>> l.append(l)
- >>> del l
-
-In this example, we create a list that contains itself. When we delete it, it
-still has a reference from itself. Its reference count doesn't drop to zero.
-Fortunately, Python's cyclic-garbage collector will eventually figure out that
-the list is garbage and free it.
-
-In the second version of the :class:`Noddy` example, we allowed any kind of
-object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This
-means that :class:`Noddy` objects can participate in cycles::
-
- >>> import noddy2
- >>> n = noddy2.Noddy()
- >>> l = [n]
- >>> n.first = l
-
-This is pretty silly, but it gives us an excuse to add support for the
-cyclic-garbage collector to the :class:`Noddy` example. To support cyclic
-garbage collection, types need to fill two slots and set a class flag that
-enables these slots:
-
-.. literalinclude:: ../includes/noddy4.c
-
-
-The traversal method provides access to subobjects that could participate in
-cycles::
-
- static int
- Noddy_traverse(Noddy *self, visitproc visit, void *arg)
- {
- int vret;
-
- if (self->first) {
- vret = visit(self->first, arg);
- if (vret != 0)
- return vret;
- }
- if (self->last) {
- vret = visit(self->last, arg);
- if (vret != 0)
- return vret;
- }
-
- return 0;
- }
-
-For each subobject that can participate in cycles, we need to call the
-:c:func:`visit` function, which is passed to the traversal method. The
-:c:func:`visit` function takes as arguments the subobject and the extra argument
-*arg* passed to the traversal method. It returns an integer value that must be
-returned if it is non-zero.
-
-Python provides a :c:func:`Py_VISIT` macro that automates calling visit
-functions. With :c:func:`Py_VISIT`, :c:func:`Noddy_traverse` can be simplified::
-
- static int
- Noddy_traverse(Noddy *self, visitproc visit, void *arg)
- {
- Py_VISIT(self->first);
- Py_VISIT(self->last);
- return 0;
- }
-
-.. note::
-
- Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly
- *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage
- uniformity across these boring implementations.
-
-We also need to provide a method for clearing any subobjects that can
-participate in cycles. We implement the method and reimplement the deallocator
-to use it::
-
- static int
- Noddy_clear(Noddy *self)
- {
- PyObject *tmp;
-
- tmp = self->first;
- self->first = NULL;
- Py_XDECREF(tmp);
-
- tmp = self->last;
- self->last = NULL;
- Py_XDECREF(tmp);
-
- return 0;
- }
-
- static void
- Noddy_dealloc(Noddy* self)
- {
- Noddy_clear(self);
- Py_TYPE(self)->tp_free((PyObject*)self);
- }
-
-Notice the use of a temporary variable in :c:func:`Noddy_clear`. We use the
-temporary variable so that we can set each member to *NULL* before decrementing
-its reference count. We do this because, as was discussed earlier, if the
-reference count drops to zero, we might cause code to run that calls back into
-the object. In addition, because we now support garbage collection, we also
-have to worry about code being run that triggers garbage collection. If garbage
-collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't
-take a chance of having :c:func:`Noddy_traverse` called when a member's reference
-count has dropped to zero and its value hasn't been set to *NULL*.
-
-Python provides a :c:func:`Py_CLEAR` that automates the careful decrementing of
-reference counts. With :c:func:`Py_CLEAR`, the :c:func:`Noddy_clear` function can
-be simplified::
-
- static int
- Noddy_clear(Noddy *self)
- {
- Py_CLEAR(self->first);
- Py_CLEAR(self->last);
- return 0;
- }
-
-Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
-
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */
-
-That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or
-:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection.
-Most extensions will use the versions automatically provided.
-
-
-Subclassing other types
------------------------
-
-It is possible to create new extension types that are derived from existing
-types. It is easiest to inherit from the built in types, since an extension can
-easily use the :class:`PyTypeObject` it needs. It can be difficult to share
-these :class:`PyTypeObject` structures between extension modules.
-
-In this example we will create a :class:`Shoddy` type that inherits from the
-built-in :class:`list` type. The new type will be completely compatible with
-regular lists, but will have an additional :meth:`increment` method that
-increases an internal counter. ::
-
- >>> import shoddy
- >>> s = shoddy.Shoddy(range(3))
- >>> s.extend(s)
- >>> print(len(s))
- 6
- >>> print(s.increment())
- 1
- >>> print(s.increment())
- 2
-
-.. literalinclude:: ../includes/shoddy.c
-
-
-As you can see, the source code closely resembles the :class:`Noddy` examples in
-previous sections. We will break down the main differences between them. ::
-
- typedef struct {
- PyListObject list;
- int state;
- } Shoddy;
-
-The primary difference for derived type objects is that the base type's object
-structure must be the first value. The base type will already include the
-:c:func:`PyObject_HEAD` at the beginning of its structure.
-
-When a Python object is a :class:`Shoddy` instance, its *PyObject\** pointer can
-be safely cast to both *PyListObject\** and *Shoddy\**. ::
-
- static int
- Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds)
- {
- if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0)
- return -1;
- self->state = 0;
- return 0;
- }
-
-In the :attr:`__init__` method for our type, we can see how to call through to
-the :attr:`__init__` method of the base type.
-
-This pattern is important when writing a type with custom :attr:`new` and
-:attr:`dealloc` methods. The :attr:`new` method should not actually create the
-memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base
-class when calling its :c:member:`~PyTypeObject.tp_new`.
-
-When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see
-a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't
-fill that field directly with the :c:func:`PyList_Type`; it can be done later in
-the module's :c:func:`init` function. ::
-
- PyMODINIT_FUNC
- PyInit_shoddy(void)
- {
- PyObject *m;
-
- ShoddyType.tp_base = &PyList_Type;
- if (PyType_Ready(&ShoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&shoddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&ShoddyType);
- PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType);
- return m;
- }
-
-Before calling :c:func:`PyType_Ready`, the type structure must have the
-:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not
-necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew`
--- the allocate function from the base type will be inherited.
-
-After that, calling :c:func:`PyType_Ready` and adding the type object to the
-module is the same as with the basic :class:`Noddy` examples.
-
+*****************************************
+Defining Extension Types: Assorted Topics
+*****************************************
.. _dnt-type-methods:
-Type Methods
-============
-
This section aims to give a quick fly-by on the various type methods you can
implement and what they do.
@@ -889,21 +15,20 @@ debug builds omitted:
.. literalinclude:: ../includes/typestruct.h
-Now that's a *lot* of methods. Don't worry too much though - if you have a type
-you want to define, the chances are very good that you will only implement a
-handful of these.
+Now that's a *lot* of methods. Don't worry too much though -- if you have
+a type you want to define, the chances are very good that you will only
+implement a handful of these.
As you probably expect by now, we're going to go over this and give more
information about the various handlers. We won't go in the order they are
defined in the structure, because there is a lot of historical baggage that
-impacts the ordering of the fields; be sure your type initialization keeps the
-fields in the right order! It's often easiest to find an example that includes
-all the fields you need (even if they're initialized to ``0``) and then change
-the values to suit your new type. ::
+impacts the ordering of the fields. It's often easiest to find an example
+that includes the fields you need and then change the values to suit your new
+type. ::
const char *tp_name; /* For printing */
-The name of the type - as mentioned in the last section, this will appear in
+The name of the type -- as mentioned in the previous chapter, this will appear in
various places, almost entirely for diagnostic purposes. Try to choose something
that will be helpful in such a situation! ::
@@ -911,7 +36,7 @@ that will be helpful in such a situation! ::
These fields tell the runtime how much memory to allocate when new objects of
this type are created. Python has some built-in support for variable length
-structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field
+structures (think: strings, tuples) which is where the :c:member:`~PyTypeObject.tp_itemsize` field
comes in. This will be dealt with later. ::
const char *tp_doc;
@@ -919,7 +44,7 @@ comes in. This will be dealt with later. ::
Here you can put a string (or its address) that you want returned when the
Python script references ``obj.__doc__`` to retrieve the doc string.
-Now we come to the basic type methods---the ones most extension types will
+Now we come to the basic type methods -- the ones most extension types will
implement.
@@ -943,7 +68,7 @@ object itself needs to be freed here as well. Here is an example of this
function::
static void
- newdatatype_dealloc(newdatatypeobject * obj)
+ newdatatype_dealloc(newdatatypeobject *obj)
{
free(obj->obj_UnderlyingDatatypePtr);
Py_TYPE(obj)->tp_free(obj);
@@ -1031,7 +156,7 @@ example::
static PyObject *
newdatatype_repr(newdatatypeobject * obj)
{
- return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:\%d}}",
+ return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
}
@@ -1051,7 +176,7 @@ Here is a simple example::
static PyObject *
newdatatype_str(newdatatypeobject * obj)
{
- return PyUnicode_FromFormat("Stringified_newdatatype{{size:\%d}}",
+ return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}",
obj->obj_UnderlyingDatatypePtr->size);
}
@@ -1232,7 +357,7 @@ example that simply raises an exception; if this were really all you wanted, the
static int
newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v)
{
- (void)PyErr_Format(PyExc_RuntimeError, "Read-only attribute: \%s", name);
+ PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name);
return -1;
}
@@ -1317,17 +442,23 @@ these in the :file:`Objects` directory of the Python source distribution. ::
hashfunc tp_hash;
This function, if you choose to provide it, should return a hash number for an
-instance of your data type. Here is a moderately pointless example::
+instance of your data type. Here is a simple example::
- static long
+ static Py_hash_t
newdatatype_hash(newdatatypeobject *obj)
{
- long result;
- result = obj->obj_UnderlyingDatatypePtr->size;
- result = result * 3;
+ Py_hash_t result;
+ result = obj->some_size + 32767 * obj->some_number;
+ if (result == -1)
+ result = -2;
return result;
}
+:c:type:`Py_hash_t` is a signed integer type with a platform-varying width.
+Returning ``-1`` from :c:member:`~PyTypeObject.tp_hash` indicates an error,
+which is why you should be careful to avoid returning it when hash computation
+is successful, as seen above.
+
::
ternaryfunc tp_call;
@@ -1338,27 +469,22 @@ contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is inv
This function takes three arguments:
-#. *arg1* is the instance of the data type which is the subject of the call. If
- the call is ``obj1('hello')``, then *arg1* is ``obj1``.
+#. *self* is the instance of the data type which is the subject of the call.
+ If the call is ``obj1('hello')``, then *self* is ``obj1``.
-#. *arg2* is a tuple containing the arguments to the call. You can use
+#. *args* is a tuple containing the arguments to the call. You can use
:c:func:`PyArg_ParseTuple` to extract the arguments.
-#. *arg3* is a dictionary of keyword arguments that were passed. If this is
+#. *kwds* is a dictionary of keyword arguments that were passed. If this is
non-*NULL* and you support keyword arguments, use
- :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you do not
- want to support keyword arguments and this is non-*NULL*, raise a
+ :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you
+ do not want to support keyword arguments and this is non-*NULL*, raise a
:exc:`TypeError` with a message saying that keyword arguments are not supported.
-Here is a desultory example of the implementation of the call function. ::
+Here is a toy ``tp_call`` implementation::
- /* Implement the call function.
- * obj1 is the instance receiving the call.
- * obj2 is a tuple containing the arguments to the call, in this
- * case 3 strings.
- */
static PyObject *
- newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *other)
+ newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds)
{
PyObject *result;
char *arg1;
@@ -1369,7 +495,7 @@ Here is a desultory example of the implementation of the call function. ::
return NULL;
}
result = PyUnicode_FromFormat(
- "Returning -- value: [\%d] arg1: [\%s] arg2: [\%s] arg3: [\%s]\n",
+ "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n",
obj->obj_UnderlyingDatatypePtr->size,
arg1, arg2, arg3);
return result;
@@ -1381,32 +507,36 @@ Here is a desultory example of the implementation of the call function. ::
getiterfunc tp_iter;
iternextfunc tp_iternext;
-These functions provide support for the iterator protocol. Any object which
-wishes to support iteration over its contents (which may be generated during
-iteration) must implement the ``tp_iter`` handler. Objects which are returned
-by a ``tp_iter`` handler must implement both the ``tp_iter`` and ``tp_iternext``
-handlers. Both handlers take exactly one parameter, the instance for which they
-are being called, and return a new reference. In the case of an error, they
-should set an exception and return *NULL*.
-
-For an object which represents an iterable collection, the ``tp_iter`` handler
-must return an iterator object. The iterator object is responsible for
-maintaining the state of the iteration. For collections which can support
-multiple iterators which do not interfere with each other (as lists and tuples
-do), a new iterator should be created and returned. Objects which can only be
-iterated over once (usually due to side effects of iteration) should implement
-this handler by returning a new reference to themselves, and should also
-implement the ``tp_iternext`` handler. File objects are an example of such an
-iterator.
-
-Iterator objects should implement both handlers. The ``tp_iter`` handler should
-return a new reference to the iterator (this is the same as the ``tp_iter``
-handler for objects which can only be iterated over destructively). The
-``tp_iternext`` handler should return a new reference to the next object in the
-iteration if there is one. If the iteration has reached the end, it may return
-*NULL* without setting an exception or it may set :exc:`StopIteration`; avoiding
-the exception can yield slightly better performance. If an actual error occurs,
-it should set an exception and return *NULL*.
+These functions provide support for the iterator protocol. Both handlers
+take exactly one parameter, the instance for which they are being called,
+and return a new reference. In the case of an error, they should set an
+exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds
+to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext`
+corresponds to the Python :meth:`~iterator.__next__` method.
+
+Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter`
+handler, which must return an :term:`iterator` object. Here the same guidelines
+apply as for Python classes:
+
+* For collections (such as lists and tuples) which can support multiple
+ independent iterators, a new iterator should be created and returned by
+ each call to :c:member:`~PyTypeObject.tp_iter`.
+* Objects which can only be iterated over once (usually due to side effects of
+ iteration, such as file objects) can implement :c:member:`~PyTypeObject.tp_iter`
+ by returning a new reference to themselves -- and should also therefore
+ implement the :c:member:`~PyTypeObject.tp_iternext` handler.
+
+Any :term:`iterator` object should implement both :c:member:`~PyTypeObject.tp_iter`
+and :c:member:`~PyTypeObject.tp_iternext`. An iterator's
+:c:member:`~PyTypeObject.tp_iter` handler should return a new reference
+to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should
+return a new reference to the next object in the iteration, if there is one.
+If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext`
+may return *NULL* without setting an exception, or it may set
+:exc:`StopIteration` *in addition* to returning *NULL*; avoiding
+the exception can yield slightly better performance. If an actual error
+occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception
+and return *NULL*.
.. _weakref-support:
@@ -1414,110 +544,76 @@ it should set an exception and return *NULL*.
Weak Reference Support
----------------------
-One of the goals of Python's weak-reference implementation is to allow any type
+One of the goals of Python's weak reference implementation is to allow any type
to participate in the weak reference mechanism without incurring the overhead on
-those objects which do not benefit by weak referencing (such as numbers).
+performance-critical objects (such as numbers).
-For an object to be weakly referencable, the extension must include a
-:c:type:`PyObject\*` field in the instance structure for the use of the weak
-reference mechanism; it must be initialized to *NULL* by the object's
-constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the
-corresponding type object to the offset of the field. For example, the instance
-type is defined with the following structure::
+.. seealso::
+ Documentation for the :mod:`weakref` module.
- typedef struct {
- PyObject_HEAD
- PyClassObject *in_class; /* The class object */
- PyObject *in_dict; /* A dictionary */
- PyObject *in_weakreflist; /* List of weak references */
- } PyInstanceObject;
-
-The statically-declared type object for instances is defined this way::
-
- PyTypeObject PyInstance_Type = {
- PyVarObject_HEAD_INIT(&PyType_Type, 0)
- 0,
- "module.instance",
-
- /* Lots of stuff omitted for brevity... */
-
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- 0, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- offsetof(PyInstanceObject, in_weakreflist), /* tp_weaklistoffset */
- };
+For an object to be weakly referencable, the extension type must do two things:
-The type constructor is responsible for initializing the weak reference list to
-*NULL*::
+#. Include a :c:type:`PyObject\*` field in the C object structure dedicated to
+ the weak reference mechanism. The object's constructor should leave it
+ *NULL* (which is automatic when using the default
+ :c:member:`~PyTypeObject.tp_alloc`).
- static PyObject *
- instance_new() {
- /* Other initialization stuff omitted for brevity */
+#. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member
+ to the offset of the aforementioned field in the C object structure,
+ so that the interpreter knows how to access and modify that field.
- self->in_weakreflist = NULL;
+Concretely, here is how a trivial object structure would be augmented
+with the required field::
- return (PyObject *) self;
- }
+ typedef struct {
+ PyObject_HEAD
+ PyObject *weakreflist; /* List of weak references */
+ } TrivialObject;
-The only further addition is that the destructor needs to call the weak
-reference manager to clear any weak references. This is only required if the
-weak reference list is non-*NULL*::
+And the corresponding member in the statically-declared type object::
- static void
- instance_dealloc(PyInstanceObject *inst)
- {
- /* Allocate temporaries if needed, but do not begin
- destruction just yet.
- */
+ static PyTypeObject TrivialType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ /* ... other members omitted for brevity ... */
+ .tp_weaklistoffset = offsetof(TrivialObject, weakreflist),
+ };
- if (inst->in_weakreflist != NULL)
- PyObject_ClearWeakRefs((PyObject *) inst);
+The only further addition is that ``tp_dealloc`` needs to clear any weak
+references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is
+non-*NULL*::
- /* Proceed with object destruction normally. */
+ static void
+ Trivial_dealloc(TrivialObject *self)
+ {
+ /* Clear weakrefs first before calling any destructors */
+ if (self->weakreflist != NULL)
+ PyObject_ClearWeakRefs((PyObject *) self);
+ /* ... remainder of destruction code omitted for brevity ... */
+ Py_TYPE(self)->tp_free((PyObject *) self);
}
More Suggestions
----------------
-Remember that you can omit most of these functions, in which case you provide
-``0`` as a value. There are type definitions for each of the functions you must
-provide. They are in :file:`object.h` in the Python include directory that
-comes with the source distribution of Python.
-
In order to learn how to implement any specific method for your new data type,
-do the following: Download and unpack the Python source distribution. Go to
-the :file:`Objects` directory, then search the C source files for ``tp_`` plus
-the function you want (for example, ``tp_richcompare``). You will find examples
-of the function you want to implement.
+get the :term:`CPython` source code. Go to the :file:`Objects` directory,
+then search the C source files for ``tp_`` plus the function you want
+(for example, ``tp_richcompare``). You will find examples of the function
+you want to implement.
-When you need to verify that an object is an instance of the type you are
-implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of its use
-might be something like the following::
+When you need to verify that an object is a concrete instance of the type you
+are implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of
+its use might be something like the following::
- if (! PyObject_TypeCheck(some_object, &MyType)) {
+ if (!PyObject_TypeCheck(some_object, &MyType)) {
PyErr_SetString(PyExc_TypeError, "arg #1 not a mything");
return NULL;
}
-.. rubric:: Footnotes
-
-.. [#] This is true when we know that the object is a basic type, like a string or a
- float.
-
-.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our
- type doesn't support garbage collection. Even if a type supports garbage
- collection, there are calls that can be made to "untrack" the object from
- garbage collection, however, these calls are advanced and not covered here.
-
-.. [#] We now know that the first and last members are strings, so perhaps we could be
- less careful about decrementing their reference counts, however, we accept
- instances of string subclasses. Even though deallocating normal strings won't
- call back into our objects, we can't guarantee that deallocating an instance of
- a string subclass won't call back into our objects.
+.. seealso::
+ Download CPython source releases.
+ https://www.python.org/downloads/source/
-.. [#] Even in the third version, we aren't guaranteed to avoid cycles. Instances of
- string subclasses are allowed and string subclasses could allow cycles even if
- normal strings don't.
+ The CPython project on GitHub, where the CPython source code is developed.
+ https://github.com/python/cpython
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
new file mode 100644
index 000000000000000..ac48637bbee9bea
--- /dev/null
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -0,0 +1,896 @@
+.. highlightlang:: c
+
+.. _defining-new-types:
+
+**********************************
+Defining Extension Types: Tutorial
+**********************************
+
+.. sectionauthor:: Michael Hudson
+.. sectionauthor:: Dave Kuhlman
+.. sectionauthor:: Jim Fulton
+
+
+Python allows the writer of a C extension module to define new types that
+can be manipulated from Python code, much like the built-in :class:`str`
+and :class:`list` types. The code for all extension types follows a
+pattern, but there are some details that you need to understand before you
+can get started. This document is a gentle introduction to the topic.
+
+
+.. _dnt-basics:
+
+The Basics
+==========
+
+The :term:`CPython` runtime sees all Python objects as variables of type
+:c:type:`PyObject\*`, which serves as a "base type" for all Python objects.
+The :c:type:`PyObject` structure itself only contains the object's
+:term:`reference count` and a pointer to the object's "type object".
+This is where the action is; the type object determines which (C) functions
+get called by the interpreter when, for instance, an attribute gets looked up
+on an object, a method called, or it is multiplied by another object. These
+C functions are called "type methods".
+
+So, if you want to define a new extension type, you need to create a new type
+object.
+
+This sort of thing can only be explained by example, so here's a minimal, but
+complete, module that defines a new type named :class:`Custom` inside a C
+extension module :mod:`custom`:
+
+.. note::
+ What we're showing here is the traditional way of defining *static*
+ extension types. It should be adequate for most uses. The C API also
+ allows defining heap-allocated extension types using the
+ :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial.
+
+.. literalinclude:: ../includes/custom.c
+
+Now that's quite a bit to take in at once, but hopefully bits will seem familiar
+from the previous chapter. This file defines three things:
+
+#. What a :class:`Custom` **object** contains: this is the ``CustomObject``
+ struct, which is allocated once for each :class:`Custom` instance.
+#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct,
+ which defines a set of flags and function pointers that the interpreter
+ inspects when specific operations are requested.
+#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom``
+ function and the associated ``custommodule`` struct.
+
+The first bit is::
+
+ typedef struct {
+ PyObject_HEAD
+ } CustomObject;
+
+This is what a Custom object will contain. ``PyObject_HEAD`` is mandatory
+at the start of each object struct and defines a field called ``ob_base``
+of type :c:type:`PyObject`, containing a pointer to a type object and a
+reference count (these can be accessed using the macros :c:macro:`Py_REFCNT`
+and :c:macro:`Py_TYPE` respectively). The reason for the macro is to
+abstract away the layout and to enable additional fields in debug builds.
+
+.. note::
+ There is no semicolon above after the :c:macro:`PyObject_HEAD` macro.
+ Be wary of adding one by accident: some compilers will complain.
+
+Of course, objects generally store additional data besides the standard
+``PyObject_HEAD`` boilerplate; for example, here is the definition for
+standard Python floats::
+
+ typedef struct {
+ PyObject_HEAD
+ double ob_fval;
+ } PyFloatObject;
+
+The second bit is the definition of the type object. ::
+
+ static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_new = PyType_GenericNew,
+ };
+
+.. note::
+ We recommend using C99-style designated initializers as above, to
+ avoid listing all the :c:type:`PyTypeObject` fields that you don't care
+ about and also to avoid caring about the fields' declaration order.
+
+The actual definition of :c:type:`PyTypeObject` in :file:`object.h` has
+many more :ref:`fields ` than the definition above. The
+remaining fields will be filled with zeros by the C compiler, and it's
+common practice to not specify them explicitly unless you need them.
+
+We're going to pick it apart, one field at a time::
+
+ PyVarObject_HEAD_INIT(NULL, 0)
+
+This line is mandatory boilerplate to initialize the ``ob_base``
+field mentioned above. ::
+
+ .tp_name = "custom.Custom",
+
+The name of our type. This will appear in the default textual representation of
+our objects and in some error messages, for example:
+
+.. code-block:: pycon
+
+ >>> "" + custom.Custom()
+ Traceback (most recent call last):
+ File "", line 1, in
+ TypeError: can only concatenate str (not "custom.Custom") to str
+
+Note that the name is a dotted name that includes both the module name and the
+name of the type within the module. The module in this case is :mod:`custom` and
+the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`.
+Using the real dotted import path is important to make your type compatible
+with the :mod:`pydoc` and :mod:`pickle` modules. ::
+
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+
+This is so that Python knows how much memory to allocate when creating
+new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is
+only used for variable-sized objects and should otherwise be zero.
+
+.. note::
+
+ If you want your type to be subclassable from Python, and your type has the same
+ :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple
+ inheritance. A Python subclass of your type will have to list your type first
+ in its :attr:`~class.__bases__`, or else it will not be able to call your type's
+ :meth:`__new__` method without getting an error. You can avoid this problem by
+ ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its
+ base type does. Most of the time, this will be true anyway, because either your
+ base type will be :class:`object`, or else you will be adding data members to
+ your base type, and therefore increasing its size.
+
+We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. ::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT,
+
+All types should include this constant in their flags. It enables all of the
+members defined until at least Python 3.3. If you need further members,
+you will need to OR the corresponding flags.
+
+We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
+
+ .tp_doc = "Custom objects",
+
+To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new`
+handler. This is the equivalent of the Python method :meth:`__new__`, but
+has to be specified explicitly. In this case, we can just use the default
+implementation provided by the API function :c:func:`PyType_GenericNew`. ::
+
+ .tp_new = PyType_GenericNew,
+
+Everything else in the file should be familiar, except for some code in
+:c:func:`PyInit_custom`::
+
+ if (PyType_Ready(&CustomType) < 0)
+ return;
+
+This initializes the :class:`Custom` type, filling in a number of members
+to the appropriate default values, including :attr:`ob_type` that we initially
+set to *NULL*. ::
+
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+
+This adds the type to the module dictionary. This allows us to create
+:class:`Custom` instances by calling the :class:`Custom` class:
+
+.. code-block:: pycon
+
+ >>> import custom
+ >>> mycustom = custom.Custom()
+
+That's it! All that remains is to build it; put the above code in a file called
+:file:`custom.c` and:
+
+.. code-block:: python
+
+ from distutils.core import setup, Extension
+ setup(name="custom", version="1.0",
+ ext_modules=[Extension("custom", ["custom.c"])])
+
+in a file called :file:`setup.py`; then typing
+
+.. code-block:: shell-session
+
+ $ python setup.py build
+
+at a shell should produce a file :file:`custom.so` in a subdirectory; move to
+that directory and fire up Python --- you should be able to ``import custom`` and
+play around with Custom objects.
+
+That wasn't so hard, was it?
+
+Of course, the current Custom type is pretty uninteresting. It has no data and
+doesn't do anything. It can't even be subclassed.
+
+.. note::
+ While this documentation showcases the standard :mod:`distutils` module
+ for building C extensions, it is recommended in real-world use cases to
+ use the newer and better-maintained ``setuptools`` library. Documentation
+ on how to do this is out of scope for this document and can be found in
+ the `Python Packaging User's Guide `_.
+
+
+Adding data and methods to the Basic example
+============================================
+
+Let's extend the basic example to add some data and methods. Let's also make
+the type usable as a base class. We'll create a new module, :mod:`custom2` that
+adds these capabilities:
+
+.. literalinclude:: ../includes/custom2.c
+
+
+This version of the module has a number of changes.
+
+We've added an extra include::
+
+ #include
+
+This include provides declarations that we use to handle attributes, as
+described a bit later.
+
+The :class:`Custom` type now has three data attributes in its C struct,
+*first*, *last*, and *number*. The *first* and *last* variables are Python
+strings containing first and last names. The *number* attribute is a C integer.
+
+The object structure is updated accordingly::
+
+ typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+ } CustomObject;
+
+Because we now have data to manage, we have to be more careful about object
+allocation and deallocation. At a minimum, we need a deallocation method::
+
+ static void
+ Custom_dealloc(CustomObject *self)
+ {
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+ }
+
+which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member::
+
+ .tp_dealloc = (destructor) Custom_dealloc,
+
+This method first clears the reference counts of the two Python attributes.
+:c:func:`Py_XDECREF` correctly handles the case where its argument is
+*NULL* (which might happen here if ``tp_new`` failed midway). It then
+calls the :c:member:`~PyTypeObject.tp_free` member of the object's type
+(computed by ``Py_TYPE(self)``) to free the object's memory. Note that
+the object's type might not be :class:`CustomType`, because the object may
+be an instance of a subclass.
+
+.. note::
+ The explicit cast to ``destructor`` above is needed because we defined
+ ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc``
+ function pointer expects to receive a ``PyObject *`` argument. Otherwise,
+ the compiler will emit a warning. This is object-oriented polymorphism,
+ in C!
+
+We want to make sure that the first and last names are initialized to empty
+strings, so we provide a ``tp_new`` implementation::
+
+ static PyObject *
+ Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+ {
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+ }
+
+and install it in the :c:member:`~PyTypeObject.tp_new` member::
+
+ .tp_new = Custom_new,
+
+The ``tp_new`` handler is responsible for creating (as opposed to initializing)
+objects of the type. It is exposed in Python as the :meth:`__new__` method.
+It is not required to define a ``tp_new`` member, and indeed many extension
+types will simply reuse :c:func:`PyType_GenericNew` as done in the first
+version of the ``Custom`` type above. In this case, we use the ``tp_new``
+handler to initialize the ``first`` and ``last`` attributes to non-*NULL*
+default values.
+
+``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``,
+if a subclass is instantiated) and any arguments passed when the type was
+called, and is expected to return the instance created. ``tp_new`` handlers
+always accept positional and keyword arguments, but they often ignore the
+arguments, leaving the argument handling to initializer (a.k.a. ``tp_init``
+in C or ``__init__`` in Python) methods.
+
+.. note::
+ ``tp_new`` shouldn't call ``tp_init`` explicitly, as the interpreter
+ will do it itself.
+
+The ``tp_new`` implementation calls the :c:member:`~PyTypeObject.tp_alloc`
+slot to allocate memory::
+
+ self = (CustomObject *) type->tp_alloc(type, 0);
+
+Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc`
+result against *NULL* before proceeding.
+
+.. note::
+ We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather
+ :c:func:`PyType_Ready` fills it for us by inheriting it from our base class,
+ which is :class:`object` by default. Most types use the default allocation
+ strategy.
+
+.. note::
+ If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one
+ that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`),
+ you must *not* try to determine what method to call using method resolution
+ order at runtime. Always statically determine what type you are going to
+ call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via
+ ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your
+ type that also inherit from other Python-defined classes may not work correctly.
+ (Specifically, you may not be able to create instances of such subclasses
+ without getting a :exc:`TypeError`.)
+
+We also define an initialization function which accepts arguments to provide
+initial values for our instance::
+
+ static int
+ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+ {
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_XDECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_XDECREF(tmp);
+ }
+ return 0;
+ }
+
+by filling the :c:member:`~PyTypeObject.tp_init` slot. ::
+
+ .tp_init = (initproc) Custom_init,
+
+The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the
+:meth:`__init__` method. It is used to initialize an object after it's
+created. Initializers always accept positional and keyword arguments,
+and they should return either ``0`` on success or ``-1`` on error.
+
+Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init``
+is called at all (for example, the :mod:`pickle` module by default
+doesn't call :meth:`__init__` on unpickled instances). It can also be
+called multiple times. Anyone can call the :meth:`__init__` method on
+our objects. For this reason, we have to be extra careful when assigning
+the new attribute values. We might be tempted, for example to assign the
+``first`` member like this::
+
+ if (first) {
+ Py_XDECREF(self->first);
+ Py_INCREF(first);
+ self->first = first;
+ }
+
+But this would be risky. Our type doesn't restrict the type of the
+``first`` member, so it could be any kind of object. It could have a
+destructor that causes code to be executed that tries to access the
+``first`` member; or that destructor could release the
+:term:`Global interpreter Lock` and let arbitrary code run in other
+threads that accesses and modifies our object.
+
+To be paranoid and protect ourselves against this possibility, we almost
+always reassign members before decrementing their reference counts. When
+don't we have to do this?
+
+* when we absolutely know that the reference count is greater than 1;
+
+* when we know that deallocation of the object [#]_ will neither release
+ the :term:`GIL` nor cause any calls back into our type's code;
+
+* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc`
+ handler on a type which doesn't support cyclic garbage collection [#]_.
+
+We want to expose our instance variables as attributes. There are a
+number of ways to do that. The simplest way is to define member definitions::
+
+ static PyMemberDef Custom_members[] = {
+ {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ "first name"},
+ {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ "last name"},
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+ };
+
+and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot::
+
+ .tp_members = Custom_members,
+
+Each member definition has a member name, type, offset, access flags and
+documentation string. See the :ref:`Generic-Attribute-Management` section
+below for details.
+
+A disadvantage of this approach is that it doesn't provide a way to restrict the
+types of objects that can be assigned to the Python attributes. We expect the
+first and last names to be strings, but any Python objects can be assigned.
+Further, the attributes can be deleted, setting the C pointers to *NULL*. Even
+though we can make sure the members are initialized to non-*NULL* values, the
+members can be set to *NULL* if the attributes are deleted.
+
+We define a single method, :meth:`Custom.name()`, that outputs the objects name as the
+concatenation of the first and last names. ::
+
+ static PyObject *
+ Custom_name(CustomObject *self)
+ {
+ if (self->first == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "first");
+ return NULL;
+ }
+ if (self->last == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "last");
+ return NULL;
+ }
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+ }
+
+The method is implemented as a C function that takes a :class:`Custom` (or
+:class:`Custom` subclass) instance as the first argument. Methods always take an
+instance as the first argument. Methods often take positional and keyword
+arguments as well, but in this case we don't take any and don't need to accept
+a positional argument tuple or keyword argument dictionary. This method is
+equivalent to the Python method:
+
+.. code-block:: python
+
+ def name(self):
+ return "%s %s" % (self.first, self.last)
+
+Note that we have to check for the possibility that our :attr:`first` and
+:attr:`last` members are *NULL*. This is because they can be deleted, in which
+case they are set to *NULL*. It would be better to prevent deletion of these
+attributes and to restrict the attribute values to be strings. We'll see how to
+do that in the next section.
+
+Now that we've defined the method, we need to create an array of method
+definitions::
+
+ static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+ };
+
+(note that we used the :const:`METH_NOARGS` flag to indicate that the method
+is expecting no arguments other than *self*)
+
+and assign it to the :c:member:`~PyTypeObject.tp_methods` slot::
+
+ .tp_methods = Custom_methods,
+
+Finally, we'll make our type usable as a base class for subclassing. We've
+written our methods carefully so far so that they don't make any assumptions
+about the type of the object being created or used, so all we need to do is
+to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+
+We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the
+module name in the :c:type:`PyModuleDef` struct, and update the full class
+name in the :c:type:`PyTypeObject` struct.
+
+Finally, we update our :file:`setup.py` file to build the new module:
+
+.. code-block:: python
+
+ from distutils.core import setup, Extension
+ setup(name="custom", version="1.0",
+ ext_modules=[
+ Extension("custom", ["custom.c"]),
+ Extension("custom2", ["custom2.c"]),
+ ])
+
+
+Providing finer control over data attributes
+============================================
+
+In this section, we'll provide finer control over how the :attr:`first` and
+:attr:`last` attributes are set in the :class:`Custom` example. In the previous
+version of our module, the instance variables :attr:`first` and :attr:`last`
+could be set to non-string values or even deleted. We want to make sure that
+these attributes always contain strings.
+
+.. literalinclude:: ../includes/custom3.c
+
+
+To provide greater control, over the :attr:`first` and :attr:`last` attributes,
+we'll use custom getter and setter functions. Here are the functions for
+getting and setting the :attr:`first` attribute::
+
+ static PyObject *
+ Custom_getfirst(CustomObject *self, void *closure)
+ {
+ Py_INCREF(self->first);
+ return self->first;
+ }
+
+ static int
+ Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+ {
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ tmp = self->first;
+ Py_INCREF(value);
+ self->first = value;
+ Py_DECREF(tmp);
+ return 0;
+ }
+
+The getter function is passed a :class:`Custom` object and a "closure", which is
+a void pointer. In this case, the closure is ignored. (The closure supports an
+advanced usage in which definition data is passed to the getter and setter. This
+could, for example, be used to allow a single set of getter and setter functions
+that decide the attribute to get or set based on data in the closure.)
+
+The setter function is passed the :class:`Custom` object, the new value, and the
+closure. The new value may be *NULL*, in which case the attribute is being
+deleted. In our setter, we raise an error if the attribute is deleted or if its
+new value is not a string.
+
+We create an array of :c:type:`PyGetSetDef` structures::
+
+ static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+ };
+
+and register it in the :c:member:`~PyTypeObject.tp_getset` slot::
+
+ .tp_getset = Custom_getsetters,
+
+The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned
+above. In this case, we aren't using a closure, so we just pass *NULL*.
+
+We also remove the member definitions for these attributes::
+
+ static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+ };
+
+We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only
+allow strings [#]_ to be passed::
+
+ static int
+ Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+ {
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+ }
+
+With these changes, we can assure that the ``first`` and ``last`` members are
+never *NULL* so we can remove checks for *NULL* values in almost all cases.
+This means that most of the :c:func:`Py_XDECREF` calls can be converted to
+:c:func:`Py_DECREF` calls. The only place we can't change these calls is in
+the ``tp_dealloc`` implementation, where there is the possibility that the
+initialization of these members failed in ``tp_new``.
+
+We also rename the module initialization function and module name in the
+initialization function, as we did before, and we add an extra definition to the
+:file:`setup.py` file.
+
+
+Supporting cyclic garbage collection
+====================================
+
+Python has a :term:`cyclic garbage collector (GC) ` that
+can identify unneeded objects even when their reference counts are not zero.
+This can happen when objects are involved in cycles. For example, consider:
+
+.. code-block:: pycon
+
+ >>> l = []
+ >>> l.append(l)
+ >>> del l
+
+In this example, we create a list that contains itself. When we delete it, it
+still has a reference from itself. Its reference count doesn't drop to zero.
+Fortunately, Python's cyclic garbage collector will eventually figure out that
+the list is garbage and free it.
+
+In the second version of the :class:`Custom` example, we allowed any kind of
+object to be stored in the :attr:`first` or :attr:`last` attributes [#]_.
+Besides, in the second and third versions, we allowed subclassing
+:class:`Custom`, and subclasses may add arbitrary attributes. For any of
+those two reasons, :class:`Custom` objects can participate in cycles:
+
+.. code-block:: pycon
+
+ >>> import custom3
+ >>> class Derived(custom3.Custom): pass
+ ...
+ >>> n = Derived()
+ >>> n.some_attribute = n
+
+To allow a :class:`Custom` instance participating in a reference cycle to
+be properly detected and collected by the cyclic GC, our :class:`Custom` type
+needs to fill two additional slots and to enable a flag that enables these slots:
+
+.. literalinclude:: ../includes/custom4.c
+
+
+First, the traversal method lets the cyclic GC know about subobjects that could
+participate in cycles::
+
+ static int
+ Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+ {
+ int vret;
+ if (self->first) {
+ vret = visit(self->first, arg);
+ if (vret != 0)
+ return vret;
+ }
+ if (self->last) {
+ vret = visit(self->last, arg);
+ if (vret != 0)
+ return vret;
+ }
+ return 0;
+ }
+
+For each subobject that can participate in cycles, we need to call the
+:c:func:`visit` function, which is passed to the traversal method. The
+:c:func:`visit` function takes as arguments the subobject and the extra argument
+*arg* passed to the traversal method. It returns an integer value that must be
+returned if it is non-zero.
+
+Python provides a :c:func:`Py_VISIT` macro that automates calling visit
+functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate
+in ``Custom_traverse``::
+
+ static int
+ Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+ {
+ Py_VISIT(self->first);
+ Py_VISIT(self->last);
+ return 0;
+ }
+
+.. note::
+ The :c:member:`~PyTypeObject.tp_traverse` implementation must name its
+ arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`.
+
+Second, we need to provide a method for clearing any subobjects that can
+participate in cycles::
+
+ static int
+ Custom_clear(CustomObject *self)
+ {
+ Py_CLEAR(self->first);
+ Py_CLEAR(self->last);
+ return 0;
+ }
+
+Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe
+way to clear data attributes of arbitrary types while decrementing
+their reference counts. If you were to call :c:func:`Py_XDECREF` instead
+on the attribute before setting it to *NULL*, there is a possibility
+that the attribute's destructor would call back into code that reads the
+attribute again (*especially* if there is a reference cycle).
+
+.. note::
+ You could emulate :c:func:`Py_CLEAR` by writing::
+
+ PyObject *tmp;
+ tmp = self->first;
+ self->first = NULL;
+ Py_XDECREF(tmp);
+
+ Nevertheless, it is much easier and less error-prone to always
+ use :c:func:`Py_CLEAR` when deleting an attribute. Don't
+ try to micro-optimize at the expense of robustness!
+
+The deallocator ``Custom_dealloc`` may call arbitrary code when clearing
+attributes. It means the circular GC can be triggered inside the function.
+Since the GC assumes reference count is not zero, we need to untrack the object
+from the GC by calling :c:func:`PyObject_GC_UnTrack` before clearing members.
+Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack`
+and ``Custom_clear``::
+
+ static void
+ Custom_dealloc(CustomObject *self)
+ {
+ PyObject_GC_UnTrack(self);
+ Custom_clear(self);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+ }
+
+Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags::
+
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+
+That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or
+:c:member:`~PyTypeObject.tp_free` handlers, we'd need to modify them for cyclic
+garbage collection. Most extensions will use the versions automatically provided.
+
+
+Subclassing other types
+=======================
+
+It is possible to create new extension types that are derived from existing
+types. It is easiest to inherit from the built in types, since an extension can
+easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share
+these :c:type:`PyTypeObject` structures between extension modules.
+
+In this example we will create a :class:`SubList` type that inherits from the
+built-in :class:`list` type. The new type will be completely compatible with
+regular lists, but will have an additional :meth:`increment` method that
+increases an internal counter:
+
+.. code-block:: pycon
+
+ >>> import sublist
+ >>> s = sublist.SubList(range(3))
+ >>> s.extend(s)
+ >>> print(len(s))
+ 6
+ >>> print(s.increment())
+ 1
+ >>> print(s.increment())
+ 2
+
+.. literalinclude:: ../includes/sublist.c
+
+
+As you can see, the source code closely resembles the :class:`Custom` examples in
+previous sections. We will break down the main differences between them. ::
+
+ typedef struct {
+ PyListObject list;
+ int state;
+ } SubListObject;
+
+The primary difference for derived type objects is that the base type's
+object structure must be the first value. The base type will already include
+the :c:func:`PyObject_HEAD` at the beginning of its structure.
+
+When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer
+can be safely cast to both ``PyListObject *`` and ``SubListObject *``::
+
+ static int
+ SubList_init(SubListObject *self, PyObject *args, PyObject *kwds)
+ {
+ if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0)
+ return -1;
+ self->state = 0;
+ return 0;
+ }
+
+We see above how to call through to the :attr:`__init__` method of the base
+type.
+
+This pattern is important when writing a type with custom
+:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_dealloc`
+members. The :c:member:`~PyTypeObject.tp_new` handler should not actually
+create the memory for the object with its :c:member:`~PyTypeObject.tp_alloc`,
+but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_new`.
+
+The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base`
+specifying the type's concrete base class. Due to cross-platform compiler
+issues, you can't fill that field directly with a reference to
+:c:type:`PyList_Type`; it should be done later in the module initialization
+function::
+
+ PyMODINIT_FUNC
+ PyInit_sublist(void)
+ {
+ PyObject* m;
+ SubListType.tp_base = &PyList_Type;
+ if (PyType_Ready(&SubListType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&sublistmodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&SubListType);
+ PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+ return m;
+ }
+
+Before calling :c:func:`PyType_Ready`, the type structure must have the
+:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving an
+existing type, it is not necessary to fill out the :c:member:`~PyTypeObject.tp_alloc`
+slot with :c:func:`PyType_GenericNew` -- the allocation function from the base
+type will be inherited.
+
+After that, calling :c:func:`PyType_Ready` and adding the type object to the
+module is the same as with the basic :class:`Custom` examples.
+
+
+.. rubric:: Footnotes
+
+.. [#] This is true when we know that the object is a basic type, like a string or a
+ float.
+
+.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler
+ in this example, because our type doesn't support garbage collection.
+
+.. [#] We now know that the first and last members are strings, so perhaps we
+ could be less careful about decrementing their reference counts, however,
+ we accept instances of string subclasses. Even though deallocating normal
+ strings won't call back into our objects, we can't guarantee that deallocating
+ an instance of a string subclass won't call back into our objects.
+
+.. [#] Also, even with our attributes restricted to strings instances, the user
+ could pass arbitrary :class:`str` subclasses and therefore still create
+ reference cycles.
diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst
index 1bd800b1a81babc..234dc9c12b5ec9a 100644
--- a/Doc/faq/design.rst
+++ b/Doc/faq/design.rst
@@ -2,6 +2,11 @@
Design and History FAQ
======================
+.. only:: html
+
+ .. contents::
+
+
Why does Python use indentation for grouping of statements?
-----------------------------------------------------------
@@ -210,24 +215,25 @@ objects using the ``for`` statement. For example, :term:`file objects
Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))?
----------------------------------------------------------------------------------------------------------------
-The major reason is history. Functions were used for those operations that were
-generic for a group of types and which were intended to work even for objects
-that didn't have methods at all (e.g. tuples). It is also convenient to have a
-function that can readily be applied to an amorphous collection of objects when
-you use the functional features of Python (``map()``, ``zip()`` et al).
+As Guido said:
-In fact, implementing ``len()``, ``max()``, ``min()`` as a built-in function is
-actually less code than implementing them as methods for each type. One can
-quibble about individual cases but it's a part of Python, and it's too late to
-make such fundamental changes now. The functions have to remain to avoid massive
-code breakage.
+ (a) For some operations, prefix notation just reads better than
+ postfix -- prefix (and infix!) operations have a long tradition in
+ mathematics which likes notations where the visuals help the
+ mathematician thinking about a problem. Compare the easy with which we
+ rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of
+ doing the same thing using a raw OO notation.
-.. XXX talk about protocols?
+ (b) When I read code that says len(x) I *know* that it is asking for
+ the length of something. This tells me two things: the result is an
+ integer, and the argument is some kind of container. To the contrary,
+ when I read x.len(), I have to already know that x is some kind of
+ container implementing an interface or inheriting from a class that
+ has a standard len(). Witness the confusion we occasionally have when
+ a class that is not implementing a mapping has a get() or keys()
+ method, or something that isn't a file has a write() method.
-.. note::
-
- For string operations, Python has moved from external functions (the
- ``string`` module) to methods. However, ``len()`` is still a function.
+ -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html
Why is join() a string method instead of a list or tuple method?
@@ -465,10 +471,10 @@ you can always change a list's elements. Only immutable elements can be used as
dictionary keys, and hence only tuples and not lists can be used as keys.
-How are lists implemented?
---------------------------
+How are lists implemented in CPython?
+-------------------------------------
-Python's lists are really variable-length arrays, not Lisp-style linked lists.
+CPython's lists are really variable-length arrays, not Lisp-style linked lists.
The implementation uses a contiguous array of references to other objects, and
keeps a pointer to this array and the array's length in a list head structure.
@@ -481,10 +487,10 @@ when the array must be grown, some extra space is allocated so the next few
times don't require an actual resize.
-How are dictionaries implemented?
----------------------------------
+How are dictionaries implemented in CPython?
+--------------------------------------------
-Python's dictionaries are implemented as resizable hash tables. Compared to
+CPython's dictionaries are implemented as resizable hash tables. Compared to
B-trees, this gives better performance for lookup (the most common operation by
far) under most circumstances, and the implementation is simpler.
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index 3eafdf193c80a47..a0828172d48e250 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -53,7 +53,7 @@ with a tool such as `SWIG `_. `SIP
`__, `CXX
`_ `Boost
`_, or `Weave
-`_ are also
+`_ are also
alternatives for wrapping C++ libraries.
@@ -62,7 +62,7 @@ How can I execute arbitrary Python statements from C?
The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes
a single string argument to be executed in the context of the module
-``__main__`` and returns 0 for success and -1 when an exception occurred
+``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred
(including ``SyntaxError``). If you want more control, use
:c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in
``Python/pythonrun.c``.
@@ -277,7 +277,7 @@ However sometimes you have to run the embedded Python interpreter in the same
thread as your rest application and you can't allow the
:c:func:`PyRun_InteractiveLoop` to stop while waiting for user input. The one
solution then is to call :c:func:`PyParser_ParseString` and test for ``e.error``
-equal to ``E_EOF``, which means the input is incomplete). Here's a sample code
+equal to ``E_EOF``, which means the input is incomplete. Here's a sample code
fragment, untested, inspired by code from Alex Farber::
#include
diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst
index f1e33afdabf8a58..9ec90e802ef62b4 100644
--- a/Doc/faq/general.rst
+++ b/Doc/faq/general.rst
@@ -117,7 +117,7 @@ programming), software engineering (unit testing, logging, profiling, parsing
Python code), and operating system interfaces (system calls, filesystems, TCP/IP
sockets). Look at the table of contents for :ref:`library-index` to get an idea
of what's available. A wide variety of third-party extensions are also
-available. Consult `the Python Package Index `_ to
+available. Consult `the Python Package Index `_ to
find packages of interest to you.
@@ -159,7 +159,7 @@ How do I obtain a copy of the Python source?
The latest Python source distribution is always available from python.org, at
https://www.python.org/downloads/. The latest development sources can be obtained
-via anonymous Mercurial access at https://hg.python.org/cpython.
+at https://github.com/python/cpython/.
The source distribution is a gzipped tar file containing the complete C source,
Sphinx-formatted documentation, Python library modules, example programs, and
@@ -167,7 +167,7 @@ several useful pieces of freely distributable software. The source will compile
and run out of the box on most UNIX platforms.
Consult the `Getting Started section of the Python Developer's Guide
-`__ for more
+`__ for more
information on getting the source code and compiling it.
@@ -222,8 +222,8 @@ releases are announced on the comp.lang.python and comp.lang.python.announce
newsgroups and on the Python home page at https://www.python.org/; an RSS feed of
news is available.
-You can also access the development version of Python through Mercurial. See
-https://docs.python.org/devguide/faq.html for details.
+You can also access the development version of Python through Git. See
+`The Python Developer's Guide `_ for details.
How do I submit bug reports and patches for Python?
@@ -239,7 +239,7 @@ report bugs to Python, you can obtain your Roundup password through Roundup's
`password reset procedure `_.
For more information on how Python is developed, consult `the Python Developer's
-Guide `_.
+Guide `_.
Are there any published articles about Python that I can reference?
@@ -306,17 +306,19 @@ usually around 18 months between major releases.
The developers issue "bugfix" releases of older versions, so the stability of
existing releases gradually improves. Bugfix releases, indicated by a third
-component of the version number (e.g. 2.5.3, 2.6.2), are managed for stability;
+component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability;
only fixes for known problems are included in a bugfix release, and it's
guaranteed that interfaces will remain the same throughout a series of bugfix
releases.
The latest stable releases can always be found on the `Python download page
-`_. There are two recommended production-ready
-versions at this point in time, because at the moment there are two branches of
-stable releases: 2.x and 3.x. Python 3.x may be less useful than 2.x, since
-currently there is more third party software available for Python 2 than for
-Python 3. Python 2 code will generally not run unchanged in Python 3.
+`_. There are two production-ready version
+of Python: 2.x and 3.x, but the recommended one at this times is Python 3.x.
+Although Python 2.x is still widely used, `it will not be
+maintained after January 1, 2020 `_.
+Python 2.x was known for having more third-party libraries available, however,
+by the time of this writing, most of the widely used libraries support Python 3.x,
+and some are even dropping the Python 2.x support.
How many people are using Python?
diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst
index 477d8c633438d1a..38e1796267ff3a7 100644
--- a/Doc/faq/gui.rst
+++ b/Doc/faq/gui.rst
@@ -94,15 +94,6 @@ Python bindings for `the FLTK toolkit `_, a simple yet
powerful and mature cross-platform windowing system, are available from `the
PyFLTK project `_.
-
-FOX
-----
-
-A wrapper for `the FOX toolkit `_ called `FXpy
-`_ is available. FOX supports both Unix variants
-and Windows.
-
-
OpenGL
------
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index b5fdfa42cdb0823..1acfc26801ca22f 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -19,7 +19,7 @@ standard library module. (Eventually you'll learn what's in the standard
library and will be able to skip this step.)
For third-party packages, search the `Python Package Index
-`_ or try `Google `_ or
+`_ or try `Google `_ or
another Web search engine. Searching for "Python" plus a keyword or two for
your topic of interest will usually find something helpful.
@@ -74,7 +74,9 @@ interpreter.
Occasionally, a user's environment is so full that the :program:`/usr/bin/env`
program fails; or there's no env program at all. In that case, you can try the
-following hack (due to Alex Rezinsky)::
+following hack (due to Alex Rezinsky):
+
+.. code-block:: sh
#! /bin/sh
""":"
@@ -609,7 +611,7 @@ use ``p.read(n)``.
"expect" library. A Python extension that interfaces to expect is called
"expy" and available from http://expectpy.sourceforge.net. A pure Python
solution that works like expect is `pexpect
- `_.
+ `_.
How do I access the serial (RS232) port?
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 9c5e20dcadf5a07..b717ab8f0f81ef2 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -371,8 +371,8 @@ compute, a common technique is to cache the parameters and the resulting value
of each call to the function, and return the cached value if the same value is
requested again. This is called "memoizing", and can be implemented like this::
- # Callers will never provide a third parameter for this function.
- def expensive(arg1, arg2, _cache={}):
+ # Callers can only provide two parameters and optionally pass _cache by keyword
+ def expensive(arg1, arg2, *, _cache={}):
if (arg1, arg2) in _cache:
return _cache[(arg1, arg2)]
@@ -1638,7 +1638,7 @@ collected.
Despite the cycle collector, it's still a good idea to define an explicit
``close()`` method on objects to be called whenever you're done with them. The
-``close()`` method can then remove attributes that refer to subobjecs. Don't
+``close()`` method can then remove attributes that refer to subobjects. Don't
call :meth:`__del__` directly -- :meth:`__del__` should call ``close()`` and
``close()`` should make sure that it can be called more than once for the same
object.
diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst
index d7253436beaf61d..2b44f65692b85c4 100644
--- a/Doc/faq/windows.rst
+++ b/Doc/faq/windows.rst
@@ -1,5 +1,7 @@
:tocdepth: 2
+.. highlightlang:: none
+
.. _windows-faq:
=====================
@@ -21,30 +23,21 @@ This is not necessarily a straightforward question. If you are already familiar
with running programs from the Windows command line then everything will seem
obvious; otherwise, you might need a little more guidance.
-.. sidebar:: |Python Development on XP|_
- :subtitle: `Python Development on XP`_
-
- This series of screencasts aims to get you up and running with Python on
- Windows XP. The knowledge is distilled into 1.5 hours and will get you up
- and running with the right Python distribution, coding in your choice of IDE,
- and debugging and writing solid code with unit-tests.
-
-.. |Python Development on XP| image:: python-video-icon.png
-.. _`Python Development on XP`:
- http://showmedo.com/videotutorials/series?name=pythonOzsvaldPyNewbieSeries
-
Unless you use some sort of integrated development environment, you will end up
*typing* Windows commands into what is variously referred to as a "DOS window"
or "Command prompt window". Usually you can create such a window from your
-Start menu; under Windows 7 the menu selection is :menuselection:`Start -->
-Programs --> Accessories --> Command Prompt`. You should be able to recognize
+search bar by searching for ``cmd``. You should be able to recognize
when you have started such a window because you will see a Windows "command
-prompt", which usually looks like this::
+prompt", which usually looks like this:
+
+.. code-block:: doscon
C:\>
The letter may be different, and there might be other things after it, so you
-might just as easily see something like::
+might just as easily see something like:
+
+.. code-block:: doscon
D:\YourName\Projects\Python>
@@ -58,22 +51,28 @@ compiles it into bytecodes, and then executes the bytecodes to run your
program. So, how do you arrange for the interpreter to handle your Python?
First, you need to make sure that your command window recognises the word
-"python" as an instruction to start the interpreter. If you have opened a
-command window, you should try entering the command ``python`` and hitting
-return.::
+"py" as an instruction to start the interpreter. If you have opened a
+command window, you should try entering the command ``py`` and hitting
+return:
+
+.. code-block:: doscon
+
+ C:\Users\YourName> py
- C:\Users\YourName> python
+You should then see something like:
-You should then see something like::
+.. code-block:: pycon
- Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32
+ Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>>
You have started the interpreter in "interactive mode". That means you can enter
Python statements or expressions interactively and have them executed or
evaluated while you wait. This is one of Python's strongest features. Check it
-by entering a few expressions of your choice and seeing the results::
+by entering a few expressions of your choice and seeing the results:
+
+.. code-block:: pycon
>>> print("Hello")
Hello
@@ -81,64 +80,33 @@ by entering a few expressions of your choice and seeing the results::
'HelloHelloHello'
Many people use the interactive mode as a convenient yet highly programmable
-calculator. When you want to end your interactive Python session, hold the :kbd:`Ctrl`
-key down while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get back to your
-Windows command prompt.
+calculator. When you want to end your interactive Python session,
+call the :func:`exit` function or hold the :kbd:`Ctrl` key down
+while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get
+back to your Windows command prompt.
You may also find that you have a Start-menu entry such as :menuselection:`Start
---> Programs --> Python 3.3 --> Python (command line)` that results in you
+--> Programs --> Python 3.x --> Python (command line)` that results in you
seeing the ``>>>`` prompt in a new window. If so, the window will disappear
-after you enter the :kbd:`Ctrl-Z` character; Windows is running a single "python"
+after you call the :func:`exit` function or enter the :kbd:`Ctrl-Z`
+character; Windows is running a single "python"
command in the window, and closes it when you terminate the interpreter.
-If the ``python`` command, instead of displaying the interpreter prompt ``>>>``,
-gives you a message like::
-
- 'python' is not recognized as an internal or external command, operable program or batch file.
-
-.. sidebar:: |Adding Python to DOS Path|_
- :subtitle: `Adding Python to DOS Path`_
-
- Python is not added to the DOS path by default. This screencast will walk
- you through the steps to add the correct entry to the `System Path`, allowing
- Python to be executed from the command-line by all users.
-
-.. |Adding Python to DOS Path| image:: python-video-icon.png
-.. _`Adding Python to DOS Path`:
- http://showmedo.com/videotutorials/video?name=960000&fromSeriesID=96
-
-
-or::
+Now that we know the ``py`` command is recognized, you can give your
+Python script to it. You'll have to give either an absolute or a
+relative path to the Python script. Let's say your Python script is
+located in your desktop and is named ``hello.py``, and your command
+prompt is nicely opened in your home directory so you're seeing something
+similar to::
- Bad command or filename
+ C:\Users\YourName>
-then you need to make sure that your computer knows where to find the Python
-interpreter. To do this you will have to modify a setting called PATH, which is
-a list of directories where Windows will look for programs.
+So now you'll ask the ``py`` command to give your script to Python by
+typing ``py`` followed by your script path::
-You should arrange for Python's installation directory to be added to the PATH
-of every command window as it starts. If you installed Python fairly recently
-then the command ::
- dir C:\py*
-
-will probably tell you where it is installed; the usual location is something
-like ``C:\Python33``. Otherwise you will be reduced to a search of your whole
-disk ... use :menuselection:`Tools --> Find` or hit the :guilabel:`Search`
-button and look for "python.exe". Supposing you discover that Python is
-installed in the ``C:\Python33`` directory (the default at the time of writing),
-you should make sure that entering the command ::
-
- c:\Python33\python
-
-starts up the interpreter as above (and don't forget you'll need a ":kbd:`Ctrl-Z`" and
-an ":kbd:`Enter`" to get out of it). Once you have verified the directory, you can
-add it to the system path to make it easier to start Python by just running
-the ``python`` command. This is currently an option in the installer as of
-CPython 3.3.
-
-More information about environment variables can be found on the
-:ref:`Using Python on Windows ` page.
+ C:\Users\YourName> py Desktop\hello.py
+ hello
How do I make Python scripts executable?
----------------------------------------
@@ -300,9 +268,10 @@ this respect, and is easily configured to use spaces: Take :menuselection:`Tools
--> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent
size" to 4, and select the "Insert spaces" radio button.
-If you suspect mixed tabs and spaces are causing problems in leading whitespace,
-run Python with the :option:`-t` switch or run ``Tools/Scripts/tabnanny.py`` to
-check a directory tree in batch mode.
+Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs
+and spaces are causing problems in leading whitespace.
+You may also run the :mod:`tabnanny` module to check a directory tree
+in batch mode.
How do I check for a keypress without blocking?
@@ -311,34 +280,3 @@ How do I check for a keypress without blocking?
Use the msvcrt module. This is a standard Windows-specific extension module.
It defines a function ``kbhit()`` which checks whether a keyboard hit is
present, and ``getch()`` which gets one character without echoing it.
-
-
-How do I emulate os.kill() in Windows?
---------------------------------------
-
-Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`::
-
- import ctypes
-
- def kill(pid):
- """kill function for Win32"""
- kernel32 = ctypes.windll.kernel32
- handle = kernel32.OpenProcess(1, 0, pid)
- return (0 != kernel32.TerminateProcess(handle, 0))
-
-In 2.7 and 3.2, :func:`os.kill` is implemented similar to the above function,
-with the additional feature of being able to send :kbd:`Ctrl+C` and :kbd:`Ctrl+Break`
-to console subprocesses which are designed to handle those signals. See
-:func:`os.kill` for further details.
-
-How do I extract the downloaded documentation on Windows?
----------------------------------------------------------
-
-Sometimes, when you download the documentation package to a Windows machine
-using a web browser, the file extension of the saved file ends up being .EXE.
-This is a mistake; the extension should be .TGZ.
-
-Simply rename the downloaded file to have the .TGZ extension, and WinZip will be
-able to handle it. (If your copy of WinZip doesn't, get a newer one from
-https://www.winzip.com.)
-
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 41ee3d83b311fa5..2aab8bf6cdc46c4 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -14,8 +14,9 @@ Glossary
``...``
The default Python prompt of the interactive shell when entering code for
- an indented code block or within a pair of matching left and right
- delimiters (parentheses, square brackets or curly braces).
+ an indented code block, when within a pair of matching left and right
+ delimiters (parentheses, square brackets, curly braces or triple quotes),
+ or after specifying a decorator.
2to3
A tool that tries to convert Python 2.x code to Python 3.x code by
@@ -39,6 +40,20 @@ Glossary
and loaders (in the :mod:`importlib.abc` module). You can create your own
ABCs with the :mod:`abc` module.
+ annotation
+ A label associated with a variable, a class
+ attribute or a function parameter or return value,
+ used by convention as a :term:`type hint`.
+
+ Annotations of local variables cannot be accessed at runtime, but
+ annotations of global variables, class attributes, and functions
+ are stored in the :attr:`__annotations__`
+ special attribute of modules, classes, and functions,
+ respectively.
+
+ See :term:`variable annotation`, :term:`function annotation`, :pep:`484`
+ and :pep:`526`, which describe this functionality.
+
argument
A value passed to a :term:`function` (or :term:`method`) when calling the
function. There are two kinds of argument:
@@ -80,7 +95,7 @@ Glossary
that it contains :keyword:`yield` expressions for producing a series of
values usable in an :keyword:`async for` loop.
- Usually refers to a asynchronous generator function, but may refer to an
+ Usually refers to an asynchronous generator function, but may refer to an
*asynchronous generator iterator* in some contexts. In cases where the
intended meaning isn't clear, using the full terms avoids ambiguity.
@@ -93,14 +108,14 @@ Glossary
This is an :term:`asynchronous iterator` which when called using the
:meth:`__anext__` method returns an awaitable object which will execute
- that the body of the asynchronous generator function until the
- next :keyword:`yield` expression.
+ the body of the asynchronous generator function until the next
+ :keyword:`yield` expression.
Each :keyword:`yield` temporarily suspends processing, remembering the
location execution state (including local variables and pending
try-statements). When the *asynchronous generator iterator* effectively
resumes with another awaitable returned by :meth:`__anext__`, it
- picks-up where it left-off. See :pep:`492` and :pep:`525`.
+ picks up where it left off. See :pep:`492` and :pep:`525`.
asynchronous iterable
An object, that can be used in an :keyword:`async for` statement.
@@ -108,10 +123,10 @@ Glossary
:meth:`__aiter__` method. Introduced by :pep:`492`.
asynchronous iterator
- An object that implements :meth:`__aiter__` and :meth:`__anext__`
+ An object that implements the :meth:`__aiter__` and :meth:`__anext__`
methods. ``__anext__`` must return an :term:`awaitable` object.
- :keyword:`async for` resolves awaitable returned from asynchronous
- iterator's :meth:`__anext__` method until it raises
+ :keyword:`async for` resolves the awaitables returned by an asynchronous
+ iterator's :meth:`__anext__` method until it raises a
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
attribute
@@ -131,9 +146,13 @@ Glossary
binary file
A :term:`file object` able to read and write
:term:`bytes-like objects `.
+ Examples of binary files are files opened in binary mode (``'rb'``,
+ ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`,
+ :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and
+ :class:`gzip.GzipFile`.
- .. seealso::
- A :term:`text file` reads and writes :class:`str` objects.
+ See also :term:`text file` for a file object able to read and write
+ :class:`str` objects.
bytes-like object
An object that supports the :ref:`bufferobjects` and can
@@ -155,7 +174,7 @@ Glossary
bytecode
Python source code is compiled into bytecode, the internal representation
of a Python program in the CPython interpreter. The bytecode is also
- cached in ``.pyc`` and ``.pyo`` files so that executing the same file is
+ cached in ``.pyc`` files so that executing the same file is
faster the second time (recompilation from source to bytecode can be
avoided). This "intermediate language" is said to run on a
:term:`virtual machine` that executes the machine code corresponding to
@@ -171,6 +190,10 @@ Glossary
normally contain method definitions which operate on instances of the
class.
+ class variable
+ A variable defined in a class and intended to be modified only at
+ class level (i.e., not in an instance of the class).
+
coercion
The implicit conversion of an instance of one type to another during an
operation which involves two arguments of the same type. For example,
@@ -316,6 +339,11 @@ Glossary
A module written in C or C++, using Python's C API to interact with the
core and with user code.
+ f-string
+ String literals prefixed with ``'f'`` or ``'F'`` are commonly called
+ "f-strings" which is short for
+ :ref:`formatted string literals `. See also :pep:`498`.
+
file object
An object exposing a file-oriented API (with methods such as
:meth:`read()` or :meth:`write()`) to an underlying resource. Depending
@@ -358,14 +386,20 @@ Glossary
and the :ref:`function` section.
function annotation
- An arbitrary metadata value associated with a function parameter or return
- value. Its syntax is explained in section :ref:`function`. Annotations
- may be accessed via the :attr:`__annotations__` special attribute of a
- function object.
+ An :term:`annotation` of a function parameter or return value.
- Python itself does not assign any particular meaning to function
- annotations. They are intended to be interpreted by third-party libraries
- or tools. See :pep:`3107`, which describes some of their potential uses.
+ Function annotations are usually used for
+ :term:`type hints `: for example, this function is expected to take two
+ :class:`int` arguments and is also expected to have an :class:`int`
+ return value::
+
+ def sum_two_numbers(a: int, b: int) -> int:
+ return a + b
+
+ Function annotation syntax is explained in section :ref:`function`.
+
+ See :term:`variable annotation` and :pep:`484`,
+ which describe this functionality.
__future__
A pseudo-module which programmers can use to enable new language features
@@ -382,7 +416,8 @@ Glossary
garbage collection
The process of freeing memory when it is not used anymore. Python
performs garbage collection via reference counting and a cyclic garbage
- collector that is able to detect and break reference cycles.
+ collector that is able to detect and break reference cycles. The
+ garbage collector can be controlled using the :mod:`gc` module.
.. index:: single: generator
@@ -401,8 +436,8 @@ Glossary
Each :keyword:`yield` temporarily suspends processing, remembering the
location execution state (including local variables and pending
- try-statements). When the *generator iterator* resumes, it picks-up where
- it left-off (in contrast to functions which start fresh on every
+ try-statements). When the *generator iterator* resumes, it picks up where
+ it left off (in contrast to functions which start fresh on every
invocation).
.. index:: single: generator expression
@@ -458,9 +493,9 @@ Glossary
Hashability makes an object usable as a dictionary key and a set member,
because these data structures use the hash value internally.
- All of Python's immutable built-in objects are hashable, while no mutable
- containers (such as lists or dictionaries) are. Objects which are
- instances of user-defined classes are hashable by default; they all
+ All of Python's immutable built-in objects are hashable; mutable
+ containers (such as lists or dictionaries) are not. Objects which are
+ instances of user-defined classes are hashable by default. They all
compare unequal (except with themselves), and their hash value is derived
from their :func:`id`.
@@ -526,7 +561,10 @@ Glossary
iterables include all sequence types (such as :class:`list`, :class:`str`,
and :class:`tuple`) and some non-sequence types like :class:`dict`,
:term:`file objects `, and objects of any classes you define
- with an :meth:`__iter__` or :meth:`__getitem__` method. Iterables can be
+ with an :meth:`__iter__` method or with a :meth:`__getitem__` method
+ that implements :term:`Sequence` semantics.
+
+ Iterables can be
used in a :keyword:`for` loop and in many other places where a sequence is
needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed
as an argument to the built-in function :func:`iter`, it returns an
@@ -582,7 +620,7 @@ Glossary
lambda
An anonymous inline function consisting of a single :term:`expression`
which is evaluated when the function is called. The syntax to create
- a lambda function is ``lambda [arguments]: expression``
+ a lambda function is ``lambda [parameters]: expression``
LBYL
Look before you leap. This coding style explicitly tests for
@@ -599,7 +637,7 @@ Glossary
list
A built-in Python :term:`sequence`. Despite its name it is more akin
to an array in other languages than to a linked list since access to
- elements are O(1).
+ elements is O(1).
list comprehension
A compact way to process all or part of the elements in a sequence and
@@ -816,6 +854,21 @@ Glossary
:class:`str` or :class:`bytes` result instead, respectively. Introduced
by :pep:`519`.
+ PEP
+ Python Enhancement Proposal. A PEP is a design document
+ providing information to the Python community, or describing a new
+ feature for Python or its processes or environment. PEPs should
+ provide a concise technical specification and a rationale for proposed
+ features.
+
+ PEPs are intended to be the primary mechanisms for proposing major new
+ features, for collecting community input on an issue, and for documenting
+ the design decisions that have gone into Python. The PEP author is
+ responsible for building consensus within the community and documenting
+ dissenting opinions.
+
+ See :pep:`1`.
+
portion
A set of files in a single directory (possibly stored in a zip file)
that contribute to a namespace package, as defined in :pep:`420`.
@@ -953,7 +1006,7 @@ Glossary
struct sequence
A tuple with named elements. Struct sequences expose an interface similar
- to :term:`named tuple` in that elements can either be accessed either by
+ to :term:`named tuple` in that elements can be accessed either by
index or as an attribute. However, they do not have any of the named tuple
methods like :meth:`~collections.somenamedtuple._make` or
:meth:`~collections.somenamedtuple._asdict`. Examples of struct sequences
@@ -966,9 +1019,12 @@ Glossary
A :term:`file object` able to read and write :class:`str` objects.
Often, a text file actually accesses a byte-oriented datastream
and handles the :term:`text encoding` automatically.
+ Examples of text files are files opened in text mode (``'r'`` or ``'w'``),
+ :data:`sys.stdin`, :data:`sys.stdout`, and instances of
+ :class:`io.StringIO`.
- .. seealso::
- A :term:`binary file` reads and write :class:`bytes` objects.
+ See also :term:`binary file` for a file object able to read and write
+ :term:`bytes-like objects `.
triple-quoted string
A string which is bound by three instances of either a quotation mark
@@ -985,6 +1041,43 @@ Glossary
:attr:`~instance.__class__` attribute or can be retrieved with
``type(obj)``.
+ type alias
+ A synonym for a type, created by assigning the type to an identifier.
+
+ Type aliases are useful for simplifying :term:`type hints `.
+ For example::
+
+ from typing import List, Tuple
+
+ def remove_gray_shades(
+ colors: List[Tuple[int, int, int]]) -> List[Tuple[int, int, int]]:
+ pass
+
+ could be made more readable like this::
+
+ from typing import List, Tuple
+
+ Color = Tuple[int, int, int]
+
+ def remove_gray_shades(colors: List[Color]) -> List[Color]:
+ pass
+
+ See :mod:`typing` and :pep:`484`, which describe this functionality.
+
+ type hint
+ An :term:`annotation` that specifies the expected type for a variable, a class
+ attribute, or a function parameter or return value.
+
+ Type hints are optional and are not enforced by Python but
+ they are useful to static type analysis tools, and aid IDEs with code
+ completion and refactoring.
+
+ Type hints of global variables, class attributes, and functions,
+ but not local variables, can be accessed using
+ :func:`typing.get_type_hints`.
+
+ See :mod:`typing` and :pep:`484`, which describe this functionality.
+
universal newlines
A manner of interpreting text streams in which all of the following are
recognized as ending a line: the Unix end-of-line convention ``'\n'``,
@@ -993,16 +1086,23 @@ Glossary
:func:`bytes.splitlines` for an additional use.
variable annotation
- A type metadata value associated with a module global variable or
- a class attribute. Its syntax is explained in section :ref:`annassign`.
- Annotations are stored in the :attr:`__annotations__` special
- attribute of a class or module object and can be accessed using
- :func:`typing.get_type_hints`.
+ An :term:`annotation` of a variable or a class attribute.
+
+ When annotating a variable or a class attribute, assignment is optional::
+
+ class C:
+ field: 'annotation'
+
+ Variable annotations are usually used for
+ :term:`type hints `: for example this variable is expected to take
+ :class:`int` values::
+
+ count: int = 0
+
+ Variable annotation syntax is explained in section :ref:`annassign`.
- Python itself does not assign any particular meaning to variable
- annotations. They are intended to be interpreted by third-party libraries
- or type checking tools. See :pep:`526`, :pep:`484` which describe
- some of their potential uses.
+ See :term:`function annotation`, :pep:`484`
+ and :pep:`526`, which describe this functionality.
virtual environment
A cooperatively isolated runtime environment that allows Python users
diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst
index 7e161a59add8aed..e78a022b372faad 100644
--- a/Doc/howto/argparse.rst
+++ b/Doc/howto/argparse.rst
@@ -24,7 +24,7 @@ Concepts
Let's show the sort of functionality that we are going to explore in this
introductory tutorial by making use of the :command:`ls` command:
-.. code-block:: sh
+.. code-block:: shell-session
$ ls
cpython devguide prog.py pypy rm-unused-function.patch
@@ -77,7 +77,7 @@ Let us start with a very simple example which does (almost) nothing::
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
$ python3 prog.py --help
@@ -119,7 +119,7 @@ An example::
And running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] echo
@@ -164,7 +164,7 @@ by reading the source code. So, let's make it a bit more useful::
And we get:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py -h
usage: prog.py [-h] echo
@@ -185,7 +185,7 @@ Now, how about doing something even more useful::
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
Traceback (most recent call last):
@@ -206,7 +206,7 @@ give it as strings, unless we tell it otherwise. So, let's tell
Following is a result of running the code:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -221,7 +221,7 @@ before proceeding.
Introducing Optional arguments
==============================
-So far we, have been playing with positional arguments. Let us
+So far we have been playing with positional arguments. Let us
have a look on how to add optional ones::
import argparse
@@ -233,7 +233,7 @@ have a look on how to add optional ones::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --verbosity 1
verbosity turned on
@@ -279,7 +279,7 @@ Let's modify the code accordingly::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --verbose
verbosity turned on
@@ -325,7 +325,7 @@ versions of the options. It's quite simple::
And here goes:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py -v
verbosity turned on
@@ -359,7 +359,7 @@ Our program keeps growing in complexity::
And now the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] [-v] square
@@ -395,7 +395,7 @@ multiple verbosity values, and actually get to use them::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -429,7 +429,7 @@ Let's fix it by restricting the values the ``--verbosity`` option can accept::
And the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 -v 3
usage: prog.py [-h] [-v {0,1,2}] square
@@ -470,7 +470,7 @@ verbosity argument (check the output of ``python --help``)::
We have introduced another action, "count",
to count the number of occurrences of a specific optional arguments:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -537,7 +537,7 @@ Let's fix::
And this is what it gives:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 -vvv
the square of 4 equals 16
@@ -581,7 +581,7 @@ it gets the ``None`` value, and that cannot be compared to an int value
And:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4
16
@@ -614,7 +614,7 @@ not just squares::
Output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py
usage: prog.py [-h] [-v] x y
@@ -652,7 +652,7 @@ to display *more* text instead::
Output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 2
16
@@ -695,7 +695,7 @@ which will be the opposite of the ``--verbose`` one::
Our program is now simpler, and we've lost some functionality for the sake of
demonstration. Anyways, here's the output:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py 4 2
4^2 == 16
@@ -739,7 +739,7 @@ Note that slight difference in the usage text. Note the ``[-v | -q]``,
which tells us that we can either use ``-v`` or ``-q``,
but not both at the same time:
-.. code-block:: sh
+.. code-block:: shell-session
$ python3 prog.py --help
usage: prog.py [-h] [-v | -q] x y
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index eaab20ad9755487..77b2aae088c3e9f 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -22,8 +22,8 @@ Argument Clinic How-To
compatibility for future versions. In other words: if you
maintain an external C extension for CPython, you're welcome
to experiment with Argument Clinic in your own code. But the
- version of Argument Clinic that ships with CPython 3.5 *could*
- be totally incompatible and break all your code.
+ version of Argument Clinic that ships with the next version
+ of CPython *could* be totally incompatible and break all your code.
The Goals Of Argument Clinic
============================
@@ -267,12 +267,16 @@ Let's dive in!
should get its own line. All the parameter lines should be
indented from the function name and the docstring.
- The general form of these parameter lines is as follows::
+ The general form of these parameter lines is as follows:
+
+ .. code-block:: none
name_of_parameter: converter
If the parameter has a default value, add that after the
- converter::
+ converter:
+
+ .. code-block:: none
name_of_parameter: converter = default_value
@@ -925,13 +929,17 @@ Parameter default values
------------------------
Default values for parameters can be any of a number of values.
-At their simplest, they can be string, int, or float literals::
+At their simplest, they can be string, int, or float literals:
+
+.. code-block:: none
foo: str = "abc"
bar: int = 123
bat: float = 45.6
-They can also use any of Python's built-in constants::
+They can also use any of Python's built-in constants:
+
+.. code-block:: none
yep: bool = True
nope: bool = False
@@ -959,7 +967,9 @@ It can be an entire expression, using math operators and looking up attributes
on objects. However, this support isn't exactly simple, because of some
non-obvious semantics.
-Consider the following example::
+Consider the following example:
+
+.. code-block:: none
foo: Py_ssize_t = sys.maxsize - 1
@@ -970,7 +980,9 @@ runtime, when the user asks for the function's signature.
What namespace is available when the expression is evaluated? It's evaluated
in the context of the module the builtin came from. So, if your module has an
-attribute called "``max_widgets``", you may simply use it::
+attribute called "``max_widgets``", you may simply use it:
+
+.. code-block:: none
foo: Py_ssize_t = max_widgets
@@ -982,7 +994,9 @@ it's best to restrict yourself to modules that are preloaded by Python itself.)
Evaluating default values only at runtime means Argument Clinic can't compute
the correct equivalent C default value. So you need to tell it explicitly.
When you use an expression, you must also specify the equivalent expression
-in C, using the ``c_default`` parameter to the converter::
+in C, using the ``c_default`` parameter to the converter:
+
+.. code-block:: none
foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1
@@ -1359,7 +1373,9 @@ Let's start with defining some terminology:
A field, in this context, is a subsection of Clinic's output.
For example, the ``#define`` for the ``PyMethodDef`` structure
is a field, called ``methoddef_define``. Clinic has seven
- different fields it can output per function definition::
+ different fields it can output per function definition:
+
+ .. code-block:: none
docstring_prototype
docstring_definition
@@ -1407,8 +1423,8 @@ Let's start with defining some terminology:
``two-pass``
A buffer like ``buffer``. However, a two-pass buffer can only
- be written once, and it prints out all text sent to it during
- all of processing, even from Clinic blocks *after* the
+ be dumped once, and it prints out all text sent to it during
+ all processing, even from Clinic blocks *after* the dumping point.
``suppress``
The text is suppressed—thrown away.
@@ -1416,7 +1432,9 @@ Let's start with defining some terminology:
Clinic defines five new directives that let you reconfigure its output.
-The first new directive is ``dump``::
+The first new directive is ``dump``:
+
+.. code-block:: none
dump
@@ -1425,7 +1443,9 @@ the current block, and empties it. This only works with ``buffer`` and
``two-pass`` destinations.
The second new directive is ``output``. The most basic form of ``output``
-is like this::
+is like this:
+
+.. code-block:: none
output
@@ -1433,7 +1453,9 @@ This tells Clinic to output *field* to *destination*. ``output`` also
supports a special meta-destination, called ``everything``, which tells
Clinic to output *all* fields to that *destination*.
-``output`` has a number of other functions::
+``output`` has a number of other functions:
+
+.. code-block:: none
output push
output pop
@@ -1471,7 +1493,7 @@ preset configurations, as follows:
The default filename is ``"{dirname}/clinic/{basename}.h"``.
``buffer``
- Save up all most of the output from Clinic, to be written into
+ Save up most of the output from Clinic, to be written into
your file near the end. For Python files implementing modules
or builtin types, it's recommended that you dump the buffer
just above the static structures for your module or
@@ -1508,7 +1530,9 @@ preset configurations, as follows:
Suppresses the ``impl_prototype``, write the ``docstring_definition``
and ``parser_definition`` to ``buffer``, write everything else to ``block``.
-The third new directive is ``destination``::
+The third new directive is ``destination``:
+
+.. code-block:: none
destination [...]
@@ -1516,7 +1540,9 @@ This performs an operation on the destination named ``name``.
There are two defined subcommands: ``new`` and ``clear``.
-The ``new`` subcommand works like this::
+The ``new`` subcommand works like this:
+
+.. code-block:: none
destination new
@@ -1564,7 +1590,9 @@ There are five destination types:
A two-pass buffer, like the "two-pass" builtin destination above.
-The ``clear`` subcommand works like this::
+The ``clear`` subcommand works like this:
+
+.. code-block:: none
destination clear
@@ -1572,7 +1600,9 @@ It removes all the accumulated text up to this point in the destination.
(I don't know what you'd need this for, but I thought maybe it'd be
useful while someone's experimenting.)
-The fourth new directive is ``set``::
+The fourth new directive is ``set``:
+
+.. code-block:: none
set line_prefix "string"
set line_suffix "string"
@@ -1590,7 +1620,9 @@ Both of these support two format strings:
Turns into the string ``*/``, the end-comment text sequence for C files.
The final new directive is one you shouldn't need to use directly,
-called ``preserve``::
+called ``preserve``:
+
+.. code-block:: none
preserve
@@ -1638,7 +1670,9 @@ like so::
#endif /* HAVE_FUNCTIONNAME */
Then, remove those three lines from the ``PyMethodDef`` structure,
-replacing them with the macro Argument Clinic generated::
+replacing them with the macro Argument Clinic generated:
+
+.. code-block:: none
MODULE_FUNCTIONNAME_METHODDEF
diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst
index 188a5cf2231b87c..d5c07714aded6ef 100644
--- a/Doc/howto/curses.rst
+++ b/Doc/howto/curses.rst
@@ -41,7 +41,7 @@ appearance---and the curses library will figure out what control codes
need to be sent to the terminal to produce the right output. curses
doesn't provide many user-interface concepts such as buttons, checkboxes,
or dialogs; if you need such features, consider a user interface library such as
-`Urwid `_.
+`Urwid `_.
The curses library was originally written for BSD Unix; the later System V
versions of Unix from AT&T added many enhancements and new functions. BSD curses
@@ -55,7 +55,7 @@ everything, though.
The Windows version of Python doesn't include the :mod:`curses`
module. A ported version called `UniCurses
-`_ is available. You could
+`_ is available. You could
also try `the Console module `_
written by Fredrik Lundh, which doesn't
use the same API as curses but provides cursor-addressable text output
@@ -65,7 +65,7 @@ and full support for mouse and keyboard input.
The Python curses module
------------------------
-Thy Python module is a fairly simple wrapper over the C functions provided by
+The Python module is a fairly simple wrapper over the C functions provided by
curses; if you're already familiar with curses programming in C, it's really
easy to transfer that knowledge to Python. The biggest difference is that the
Python interface makes things simpler by merging different C functions such as
@@ -432,7 +432,7 @@ User Input
The C curses library offers only very simple input mechanisms. Python's
:mod:`curses` module adds a basic text-input widget. (Other libraries
-such as `Urwid `_ have more extensive
+such as `Urwid `_ have more extensive
collections of widgets.)
There are two methods for getting input from a window:
@@ -538,7 +538,7 @@ the Python interface. Often this isn't because they're difficult to
implement, but because no one has needed them yet. Also, Python
doesn't yet support the menu library associated with ncurses.
Patches adding support for these would be welcome; see
-`the Python Developer's Guide `_ to
+`the Python Developer's Guide `_ to
learn more about submitting patches to Python.
* `Writing Programs with NCURSES `_:
diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst
index c2bf473e1ff9eaf..5e85a9aa6594e45 100644
--- a/Doc/howto/descriptor.rst
+++ b/Doc/howto/descriptor.rst
@@ -180,7 +180,7 @@ descriptor is useful for monitoring just a few chosen attributes::
The protocol is simple and offers exciting possibilities. Several use cases are
so common that they have been packaged into individual function calls.
-Properties, bound and unbound methods, static methods, and class methods are all
+Properties, bound methods, static methods, and class methods are all
based on the descriptor protocol.
@@ -252,10 +252,10 @@ to wrap access to the value attribute in a property data descriptor::
class Cell(object):
. . .
- def getvalue(self, obj):
- "Recalculate cell before returning value"
+ def getvalue(self):
+ "Recalculate the cell before returning value"
self.recalc()
- return obj._value
+ return self._value
value = property(getvalue)
@@ -266,23 +266,24 @@ Python's object oriented features are built upon a function based environment.
Using non-data descriptors, the two are merged seamlessly.
Class dictionaries store methods as functions. In a class definition, methods
-are written using :keyword:`def` and :keyword:`lambda`, the usual tools for
-creating functions. The only difference from regular functions is that the
+are written using :keyword:`def` or :keyword:`lambda`, the usual tools for
+creating functions. Methods only differ from regular functions in that the
first argument is reserved for the object instance. By Python convention, the
instance reference is called *self* but may be called *this* or any other
variable name.
To support method calls, functions include the :meth:`__get__` method for
binding methods during attribute access. This means that all functions are
-non-data descriptors which return bound or unbound methods depending whether
-they are invoked from an object or a class. In pure python, it works like
-this::
+non-data descriptors which return bound methods when they are invoked from an
+object. In pure python, it works like this::
class Function(object):
. . .
def __get__(self, obj, objtype=None):
"Simulate func_descr_get() in Objects/funcobject.c"
- return types.MethodType(self, obj, objtype)
+ if obj is None:
+ return self
+ return types.MethodType(self, obj)
Running the interpreter shows how the function descriptor works in practice::
@@ -291,25 +292,34 @@ Running the interpreter shows how the function descriptor works in practice::
... return x
...
>>> d = D()
- >>> D.__dict__['f'] # Stored internally as a function
-
- >>> D.f # Get from a class becomes an unbound method
-
- >>> d.f # Get from an instance becomes a bound method
+
+ # Access through the class dictionary does not invoke __get__.
+ # It just returns the underlying function object.
+ >>> D.__dict__['f']
+
+
+ # Dotted access from a class calls __get__() which just returns
+ # the underlying function unchanged.
+ >>> D.f
+
+
+ # The function has a __qualname__ attribute to support introspection
+ >>> D.f.__qualname__
+ 'D.f'
+
+ # Dotted access from an instance calls __get__() which returns the
+ # function wrapped in a bound method object
+ >>> d.f
>
-The output suggests that bound and unbound methods are two different types.
-While they could have been implemented that way, the actual C implementation of
-:c:type:`PyMethod_Type` in :source:`Objects/classobject.c` is a single object
-with two different representations depending on whether the :attr:`im_self`
-field is set or is *NULL* (the C equivalent of ``None``).
-
-Likewise, the effects of calling a method object depend on the :attr:`im_self`
-field. If set (meaning bound), the original function (stored in the
-:attr:`im_func` field) is called as expected with the first argument set to the
-instance. If unbound, all of the arguments are passed unchanged to the original
-function. The actual C implementation of :func:`instancemethod_call()` is only
-slightly more complex in that it includes some type checking.
+ # Internally, the bound method stores the underlying function,
+ # the bound instance, and the class of the bound instance.
+ >>> d.f.__func__
+
+ >>> d.f.__self__
+ <__main__.D object at 0x1012e1f98>
+ >>> d.f.__class__
+
Static Methods and Class Methods
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 8ae9679894a5785..7e21aa76a3647a7 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -198,7 +198,7 @@ for it.
You can experiment with the iteration interface manually:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> it = iter(L)
>>> it #doctest: +ELLIPSIS
<...iterator object at ...>
@@ -210,7 +210,7 @@ You can experiment with the iteration interface manually:
3
>>> next(it)
Traceback (most recent call last):
- File "", line 1, in ?
+ File "", line 1, in
StopIteration
>>>
@@ -229,7 +229,7 @@ iterator. These two statements are equivalent::
Iterators can be materialized as lists or tuples by using the :func:`list` or
:func:`tuple` constructor functions:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
>>> t = tuple(iterator)
>>> t
@@ -238,10 +238,10 @@ Iterators can be materialized as lists or tuples by using the :func:`list` or
Sequence unpacking also supports iterators: if you know an iterator will return
N elements, you can unpack them into an N-tuple:
- >>> L = [1,2,3]
+ >>> L = [1, 2, 3]
>>> iterator = iter(L)
- >>> a,b,c = iterator
- >>> a,b,c
+ >>> a, b, c = iterator
+ >>> a, b, c
(1, 2, 3)
Built-in functions such as :func:`max` and :func:`min` can take a single
@@ -410,7 +410,7 @@ lengths of all the sequences. If you have two lists of length 3, the output
list is 9 elements long:
>>> seq1 = 'abc'
- >>> seq2 = (1,2,3)
+ >>> seq2 = (1, 2, 3)
>>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE
[('a', 1), ('a', 2), ('a', 3),
('b', 1), ('b', 2), ('b', 3),
@@ -474,11 +474,11 @@ Here's a sample usage of the ``generate_ints()`` generator:
2
>>> next(gen)
Traceback (most recent call last):
- File "stdin", line 1, in ?
+ File "stdin", line 1, in
File "stdin", line 2, in generate_ints
StopIteration
-You could equally write ``for i in generate_ints(5)``, or ``a,b,c =
+You could equally write ``for i in generate_ints(5)``, or ``a, b, c =
generate_ints(3)``.
Inside a generator function, ``return value`` causes ``StopIteration(value)``
@@ -577,7 +577,7 @@ And here's an example of changing the counter:
9
>>> next(it) #doctest: +SKIP
Traceback (most recent call last):
- File "t.py", line 15, in ?
+ File "t.py", line 15, in
it.next()
StopIteration
@@ -653,8 +653,9 @@ This can also be written as a list comprehension:
[0, 2, 4, 6, 8]
-:func:`enumerate(iter) ` counts off the elements in the iterable,
-returning 2-tuples containing the count and each element. ::
+:func:`enumerate(iter, start=0) ` counts off the elements in the
+iterable returning 2-tuples containing the count (from *start*) and
+each element. ::
>>> for item in enumerate(['subject', 'verb', 'object']):
... print(item)
@@ -693,17 +694,17 @@ truth values of an iterable's contents. :func:`any` returns ``True`` if any ele
in the iterable is a true value, and :func:`all` returns ``True`` if all of the
elements are true values:
- >>> any([0,1,0])
+ >>> any([0, 1, 0])
True
- >>> any([0,0,0])
+ >>> any([0, 0, 0])
False
- >>> any([1,1,1])
+ >>> any([1, 1, 1])
True
- >>> all([0,1,0])
+ >>> all([0, 1, 0])
False
- >>> all([0,0,0])
+ >>> all([0, 0, 0])
False
- >>> all([1,1,1])
+ >>> all([1, 1, 1])
True
@@ -747,20 +748,22 @@ The module's functions fall into a few broad classes:
Creating new iterators
----------------------
-:func:`itertools.count(n) ` returns an infinite stream of
-integers, increasing by 1 each time. You can optionally supply the starting
-number, which defaults to 0::
+:func:`itertools.count(start, step) ` returns an infinite
+stream of evenly spaced values. You can optionally supply the starting number,
+which defaults to 0, and the interval between numbers, which defaults to 1::
itertools.count() =>
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...
itertools.count(10) =>
10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ...
+ itertools.count(10, 5) =>
+ 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ...
:func:`itertools.cycle(iter) ` saves a copy of the contents of
a provided iterable and returns a new iterator that returns its elements from
first to last. The new iterator will repeat these elements infinitely. ::
- itertools.cycle([1,2,3,4,5]) =>
+ itertools.cycle([1, 2, 3, 4, 5]) =>
1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ...
:func:`itertools.repeat(elem, [n]) ` returns the provided
@@ -871,7 +874,7 @@ iterable's results. ::
iterators and returns only those elements of *data* for which the corresponding
element of *selectors* is true, stopping whenever either one is exhausted::
- itertools.compress([1,2,3,4,5], [True, True, False, False, True]) =>
+ itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) =>
1, 2, 5
@@ -1031,7 +1034,7 @@ first calculation. ::
Traceback (most recent call last):
...
TypeError: reduce() of empty sequence with no initial value
- >>> functools.reduce(operator.mul, [1,2,3], 1)
+ >>> functools.reduce(operator.mul, [1, 2, 3], 1)
6
>>> functools.reduce(operator.mul, [], 1)
1
@@ -1041,9 +1044,9 @@ elements of the iterable. This case is so common that there's a special
built-in called :func:`sum` to compute it:
>>> import functools, operator
- >>> functools.reduce(operator.add, [1,2,3,4], 0)
+ >>> functools.reduce(operator.add, [1, 2, 3, 4], 0)
10
- >>> sum([1,2,3,4])
+ >>> sum([1, 2, 3, 4])
10
>>> sum([])
0
@@ -1053,22 +1056,22 @@ write the obvious :keyword:`for` loop::
import functools
# Instead of:
- product = functools.reduce(operator.mul, [1,2,3], 1)
+ product = functools.reduce(operator.mul, [1, 2, 3], 1)
# You can write:
product = 1
- for i in [1,2,3]:
+ for i in [1, 2, 3]:
product *= i
-A related function is `itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of
+returning only the final result, :func:`accumulate` returns an iterator that
+also yields each partial result::
- itertools.accumulate([1,2,3,4,5]) =>
+ itertools.accumulate([1, 2, 3, 4, 5]) =>
1, 3, 6, 10, 15
- itertools.accumulate([1,2,3,4,5], operator.mul) =>
+ itertools.accumulate([1, 2, 3, 4, 5], operator.mul) =>
1, 2, 6, 24, 120
@@ -1152,7 +1155,7 @@ But it would be best of all if I had simply used a ``for`` loop::
Or the :func:`sum` built-in and a generator expression::
- total = sum(b for a,b in items)
+ total = sum(b for a, b in items)
Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops.
@@ -1235,6 +1238,8 @@ Python documentation
Documentation for the :mod:`itertools` module.
+Documentation for the :mod:`functools` module.
+
Documentation for the :mod:`operator` module.
:pep:`289`: "Generator Expressions"
diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst
index 7fca9aac7b3a6ec..54c3fe379ac33af 100644
--- a/Doc/howto/instrumentation.rst
+++ b/Doc/howto/instrumentation.rst
@@ -254,11 +254,15 @@ and the remainder indicates the call/return hierarchy as the script executes.
For a `--enable-shared` build of CPython, the markers are contained within the
libpython shared library, and the probe's dotted path needs to reflect this. For
-example, this line from the above example::
+example, this line from the above example:
+
+.. code-block:: none
probe process("python").mark("function__entry") {
-should instead read::
+should instead read:
+
+.. code-block:: none
probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") {
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index bb79bb1748fb9c9..96d550c457b8199 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -72,7 +72,9 @@ Here is the auxiliary module::
def some_function():
module_logger.info('received a call to "some_function"')
-The output looks like this::
+The output looks like this:
+
+.. code-block:: none
2005-03-23 23:47:11,663 - spam_application - INFO -
creating an instance of auxiliary_module.Auxiliary
@@ -127,7 +129,9 @@ shows logging from the main (initial) thread and another thread::
if __name__ == '__main__':
main()
-When run, the script should print something like the following::
+When run, the script should print something like the following:
+
+.. code-block:: none
0 Thread-1 Hi from myfunc
3 MainThread Hello from main
@@ -240,14 +244,18 @@ messages should not. Here's how you can achieve this::
logger2.warning('Jail zesty vixen who grabbed pay from quack.')
logger2.error('The five boxing wizards jump quickly.')
-When you run this, on the console you will see ::
+When you run this, on the console you will see
+
+.. code-block:: none
root : INFO Jackdaws love my big sphinx of quartz.
myapp.area1 : INFO How quickly daft jumping zebras vex.
myapp.area2 : WARNING Jail zesty vixen who grabbed pay from quack.
myapp.area2 : ERROR The five boxing wizards jump quickly.
-and in the file you will see something like ::
+and in the file you will see something like
+
+.. code-block:: none
10-22 22:19 root INFO Jackdaws love my big sphinx of quartz.
10-22 22:19 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim.
@@ -515,7 +523,9 @@ module. Here is a basic working example::
main()
First run the server, and then the client. On the client side, nothing is
-printed on the console; on the server side, you should see something like::
+printed on the console; on the server side, you should see something like:
+
+.. code-block:: none
About to start TCP server...
59 root INFO Jackdaws love my big sphinx of quartz.
@@ -675,7 +685,9 @@ script::
lvlname = logging.getLevelName(lvl)
a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters')
-which, when run, produces something like::
+which, when run, produces something like:
+
+.. code-block:: none
2010-09-06 22:38:15,292 a.b.c DEBUG IP: 123.231.231.123 User: fred A debug message
2010-09-06 22:38:15,300 a.b.c INFO IP: 192.168.0.1 User: sheila An info message with some parameters
@@ -941,7 +953,7 @@ Using file rotation
-------------------
.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes)
-.. (see )
+.. (see )
Sometimes you want to let a log file grow to a certain size, then open a new
file and log to that. You may want to keep a certain number of these files, and
@@ -976,7 +988,9 @@ logging package provides a :class:`~handlers.RotatingFileHandler`::
print(filename)
The result should be 6 separate files, each with part of the log history for the
-application::
+application:
+
+.. code-block:: none
logging_rotatingfile_example.out
logging_rotatingfile_example.out.1
@@ -1258,8 +1272,8 @@ socket is created separately and passed to the handler (as its 'queue')::
class ZeroMQSocketHandler(QueueHandler):
def enqueue(self, record):
- data = json.dumps(record.__dict__)
- self.queue.send(data)
+ self.queue.send_json(record.__dict__)
+
handler = ZeroMQSocketHandler(sock)
@@ -1272,11 +1286,10 @@ data needed by the handler to create the socket::
self.ctx = ctx or zmq.Context()
socket = zmq.Socket(self.ctx, socktype)
socket.bind(uri)
- QueueHandler.__init__(self, socket)
+ super().__init__(socket)
def enqueue(self, record):
- data = json.dumps(record.__dict__)
- self.queue.send(data)
+ self.queue.send_json(record.__dict__)
def close(self):
self.queue.close()
@@ -1292,12 +1305,13 @@ of queues, for example a ZeroMQ 'subscribe' socket. Here's an example::
def __init__(self, uri, *handlers, **kwargs):
self.ctx = kwargs.get('ctx') or zmq.Context()
socket = zmq.Socket(self.ctx, zmq.SUB)
- socket.setsockopt(zmq.SUBSCRIBE, '') # subscribe to everything
+ socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything
socket.connect(uri)
+ super().__init__(socket, *handlers, **kwargs)
def dequeue(self):
- msg = self.queue.recv()
- return logging.makeLogRecord(json.loads(msg))
+ msg = self.queue.recv_json()
+ return logging.makeLogRecord(msg)
.. seealso::
@@ -1638,11 +1652,11 @@ works::
Inserting a BOM into messages sent to a SysLogHandler
-----------------------------------------------------
-`RFC 5424 `_ requires that a
+:rfc:`5424` requires that a
Unicode message be sent to a syslog daemon as a set of bytes which have the
following structure: an optional pure-ASCII component, followed by a UTF-8 Byte
-Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the `relevant
-section of the specification `_.)
+Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the
+:rfc:`relevant section of the specification <5424#section-6>`.)
In Python 3.1, code was added to
:class:`~logging.handlers.SysLogHandler` to insert a BOM into the message, but
@@ -1652,7 +1666,7 @@ appear before it.
As this behaviour is broken, the incorrect BOM insertion code is being removed
from Python 3.2.4 and later. However, it is not being replaced, and if you
-want to produce RFC 5424-compliant messages which include a BOM, an optional
+want to produce :rfc:`5424`-compliant messages which include a BOM, an optional
pure-ASCII sequence before it and arbitrary Unicode after it, encoded using
UTF-8, then you need to do the following:
@@ -1675,7 +1689,7 @@ UTF-8, then you need to do the following:
The formatted message *will* be encoded using UTF-8 encoding by
``SysLogHandler``. If you follow the above rules, you should be able to produce
-RFC 5424-compliant messages. If you don't, logging may not complain, but your
+:rfc:`5424`-compliant messages. If you don't, logging may not complain, but your
messages will not be RFC 5424-compliant, and your syslog daemon may complain.
@@ -1683,7 +1697,7 @@ Implementing structured logging
-------------------------------
Although most logging messages are intended for reading by humans, and thus not
-readily machine-parseable, there might be cirumstances where you want to output
+readily machine-parseable, there might be circumstances where you want to output
messages in a structured format which *is* capable of being parsed by a program
(without needing complex regular expressions to parse the log message). This is
straightforward to achieve using the logging package. There are a number of
@@ -1706,7 +1720,9 @@ which uses JSON to serialise the event in a machine-parseable manner::
logging.basicConfig(level=logging.INFO, format='%(message)s')
logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456))
-If the above script is run, it prints::
+If the above script is run, it prints:
+
+.. code-block:: none
message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"}
@@ -1753,7 +1769,9 @@ as in the following complete example::
if __name__ == '__main__':
main()
-When the above script is run, it prints::
+When the above script is run, it prints:
+
+.. code-block:: none
message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]}
@@ -2083,7 +2101,9 @@ most obvious, but you can provide any callable which returns a
This example shows how you can pass configuration data to the callable which
constructs the instance, in the form of keyword parameters. When run, the above
-script will print::
+script will print:
+
+.. code-block:: none
changed: hello
@@ -2150,7 +2170,9 @@ class, as shown in the following example::
if __name__ == '__main__':
main()
-When run, this produces a file with exactly two lines::
+When run, this produces a file with exactly two lines:
+
+.. code-block:: none
28/01/2015 07:21:23|INFO|Sample message|
28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'|
@@ -2312,7 +2334,9 @@ Here's the script::
write_line('Calling decorated foo with True')
assert decorated_foo(True)
-When this script is run, the following output should be observed::
+When this script is run, the following output should be observed:
+
+.. code-block:: none
Calling undecorated foo with False
about to log at DEBUG ...
@@ -2408,7 +2432,9 @@ the following complete example::
logging.config.dictConfig(LOGGING)
logging.warning('The local time is %s', time.asctime())
-When this script is run, it should print something like::
+When this script is run, it should print something like:
+
+.. code-block:: none
2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015
2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015
diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst
index 8074b0f10997d51..af8c432753cc245 100644
--- a/Doc/howto/logging.rst
+++ b/Doc/howto/logging.rst
@@ -134,7 +134,9 @@ interpreter, and don't just continue from the session described above::
logging.warning('And this, too')
And now if we open the file and look at what we have, we should find the log
-messages::
+messages:
+
+.. code-block:: none
DEBUG:root:This message should go to the log file
INFO:root:So should this
@@ -144,7 +146,9 @@ This example also shows how you can set the logging level which acts as the
threshold for tracking. In this case, because we set the threshold to
``DEBUG``, all of the messages were printed.
-If you want to set the logging level from a command-line option such as::
+If you want to set the logging level from a command-line option such as:
+
+.. code-block:: none
--log=INFO
@@ -208,7 +212,9 @@ could organize logging in it::
def do_something():
logging.info('Doing something')
-If you run *myapp.py*, you should see this in *myapp.log*::
+If you run *myapp.py*, you should see this in *myapp.log*:
+
+.. code-block:: none
INFO:root:Started
INFO:root:Doing something
@@ -258,7 +264,9 @@ specify the format you want to use::
logging.info('So should this')
logging.warning('And this, too')
-which would print::
+which would print:
+
+.. code-block:: none
DEBUG:This message should appear on the console
INFO:So should this
@@ -282,19 +290,23 @@ your format string::
logging.basicConfig(format='%(asctime)s %(message)s')
logging.warning('is when this event was logged.')
-which should print something like this::
+which should print something like this:
+
+.. code-block:: none
2010-12-12 11:41:42,612 is when this event was logged.
-The default format for date/time display (shown above) is ISO8601. If you need
-more control over the formatting of the date/time, provide a *datefmt*
-argument to ``basicConfig``, as in this example::
+The default format for date/time display (shown above) is like ISO8601 or
+:rfc:`3339`. If you need more control over the formatting of the date/time, provide
+a *datefmt* argument to ``basicConfig``, as in this example::
import logging
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.warning('is when this event was logged.')
-which would display something like this::
+which would display something like this:
+
+.. code-block:: none
12/12/2010 11:46:36 AM is when this event was logged.
@@ -376,7 +388,9 @@ if no destination is set; and if one is not set, they will set a destination
of the console (``sys.stderr``) and a default format for the displayed
message before delegating to the root logger to do the actual message output.
-The default format set by :func:`basicConfig` for messages is::
+The default format set by :func:`basicConfig` for messages is:
+
+.. code-block:: none
severity:logger name:message
@@ -522,7 +536,9 @@ indicator.
.. method:: logging.Formatter.__init__(fmt=None, datefmt=None, style='%')
If there is no message format string, the default is to use the
-raw message. If there is no date format string, the default date format is::
+raw message. If there is no date format string, the default date format is:
+
+.. code-block:: none
%Y-%m-%d %H:%M:%S
@@ -628,7 +644,9 @@ the names of the objects::
logger.error('error message')
logger.critical('critical message')
-Here is the logging.conf file::
+Here is the logging.conf file:
+
+.. code-block:: ini
[loggers]
keys=root,simpleExample
@@ -713,7 +731,9 @@ construct the dictionary in Python code, receive it in pickled form over a
socket, or use whatever approach makes sense for your application.
Here's an example of the same configuration as above, in YAML format for
-the new dictionary-based approach::
+the new dictionary-based approach:
+
+.. code-block:: yaml
version: 1
formatters:
@@ -951,7 +971,7 @@ provided:
The :class:`NullHandler`, :class:`StreamHandler` and :class:`FileHandler`
classes are defined in the core logging package. The other handlers are
-defined in a sub- module, :mod:`logging.handlers`. (There is also another
+defined in a sub-module, :mod:`logging.handlers`. (There is also another
sub-module, :mod:`logging.config`, for configuration functionality.)
Logged messages are formatted for presentation through instances of the
diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst
index 8562d2373749f08..88b01779d2a5f49 100644
--- a/Doc/howto/pyporting.rst
+++ b/Doc/howto/pyporting.rst
@@ -427,25 +427,25 @@ to make sure everything functions as expected in both versions of Python.
.. _2to3: https://docs.python.org/3/library/2to3.html
-.. _caniusepython3: https://pypi.python.org/pypi/caniusepython3
+.. _caniusepython3: https://pypi.org/project/caniusepython3
.. _cheat sheet: http://python-future.org/compatible_idioms.html
-.. _coverage.py: https://pypi.python.org/pypi/coverage
+.. _coverage.py: https://pypi.org/project/coverage
.. _Futurize: http://python-future.org/automatic_conversion.html
.. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib
-.. _importlib2: https://pypi.python.org/pypi/importlib2
+.. _importlib2: https://pypi.org/project/importlib2
.. _Modernize: https://python-modernize.readthedocs.org/en/latest/
.. _mypy: http://mypy-lang.org/
.. _Porting to Python 3: http://python3porting.com/
-.. _Pylint: https://pypi.python.org/pypi/pylint
+.. _Pylint: https://pypi.org/project/pylint
.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.org/en/latest/python3/questions_and_answers.html
.. _pytype: https://github.com/google/pytype
.. _python-future: http://python-future.org/
.. _python-porting: https://mail.python.org/mailman/listinfo/python-porting
-.. _six: https://pypi.python.org/pypi/six
-.. _tox: https://pypi.python.org/pypi/tox
-.. _trove classifier: https://pypi.python.org/pypi?%3Aaction=list_classifiers
+.. _six: https://pypi.org/project/six
+.. _tox: https://pypi.org/project/tox
+.. _trove classifier: https://pypi.org/classifiers
.. _"What's New": https://docs.python.org/3/whatsnew/index.html
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index d9b7c9091d467a1..65b09459357b5c7 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -153,8 +153,8 @@ These sequences can be included inside a character class. For example,
``','`` or ``'.'``.
The final metacharacter in this section is ``.``. It matches anything except a
-newline character, and there's an alternate mode (``re.DOTALL``) where it will
-match even a newline. ``'.'`` is often used where you want to match "any
+newline character, and there's an alternate mode (:const:`re.DOTALL`) where it will
+match even a newline. ``.`` is often used where you want to match "any
character".
@@ -168,14 +168,11 @@ wouldn't be much of an advance. Another capability is that you can specify that
portions of the RE must be repeated a certain number of times.
The first metacharacter for repeating things that we'll look at is ``*``. ``*``
-doesn't match the literal character ``*``; instead, it specifies that the
+doesn't match the literal character ``'*'``; instead, it specifies that the
previous character can be matched zero or more times, instead of exactly once.
-For example, ``ca*t`` will match ``ct`` (0 ``a`` characters), ``cat`` (1 ``a``),
-``caaat`` (3 ``a`` characters), and so forth. The RE engine has various
-internal limitations stemming from the size of C's ``int`` type that will
-prevent it from matching over 2 billion ``a`` characters; patterns
-are usually not written to match that much data.
+For example, ``ca*t`` will match ``'ct'`` (0 ``'a'`` characters), ``'cat'`` (1 ``'a'``),
+``'caaat'`` (3 ``'a'`` characters), and so forth.
Repetitions such as ``*`` are :dfn:`greedy`; when repeating a RE, the matching
engine will try to repeat it as many times as possible. If later portions of the
@@ -185,7 +182,7 @@ fewer repetitions.
A step-by-step example will make this more obvious. Let's consider the
expression ``a[bcd]*b``. This matches the letter ``'a'``, zero or more letters
from the class ``[bcd]``, and finally ends with a ``'b'``. Now imagine matching
-this RE against the string ``abcbd``.
+this RE against the string ``'abcbd'``.
+------+-----------+---------------------------------+
| Step | Matched | Explanation |
@@ -218,7 +215,7 @@ this RE against the string ``abcbd``.
| | | it succeeds. |
+------+-----------+---------------------------------+
-The end of the RE has now been reached, and it has matched ``abcb``. This
+The end of the RE has now been reached, and it has matched ``'abcb'``. This
demonstrates how the matching engine goes as far as it can at first, and if no
match is found it will then progressively back up and retry the rest of the RE
again and again. It will back up until it has tried zero matches for
@@ -229,24 +226,23 @@ Another repeating metacharacter is ``+``, which matches one or more times. Pay
careful attention to the difference between ``*`` and ``+``; ``*`` matches
*zero* or more times, so whatever's being repeated may not be present at all,
while ``+`` requires at least *one* occurrence. To use a similar example,
-``ca+t`` will match ``cat`` (1 ``a``), ``caaat`` (3 ``a``'s), but won't match
-``ct``.
+``ca+t`` will match ``'cat'`` (1 ``'a'``), ``'caaat'`` (3 ``'a'``\ s), but won't
+match ``'ct'``.
There are two more repeating qualifiers. The question mark character, ``?``,
matches either once or zero times; you can think of it as marking something as
-being optional. For example, ``home-?brew`` matches either ``homebrew`` or
-``home-brew``.
+being optional. For example, ``home-?brew`` matches either ``'homebrew'`` or
+``'home-brew'``.
The most complicated repeated qualifier is ``{m,n}``, where *m* and *n* are
decimal integers. This qualifier means there must be at least *m* repetitions,
-and at most *n*. For example, ``a/{1,3}b`` will match ``a/b``, ``a//b``, and
-``a///b``. It won't match ``ab``, which has no slashes, or ``a////b``, which
+and at most *n*. For example, ``a/{1,3}b`` will match ``'a/b'``, ``'a//b'``, and
+``'a///b'``. It won't match ``'ab'``, which has no slashes, or ``'a////b'``, which
has four.
You can omit either *m* or *n*; in that case, a reasonable value is assumed for
the missing value. Omitting *m* is interpreted as a lower limit of 0, while
-omitting *n* results in an upper bound of infinity --- actually, the upper bound
-is the 2-billion limit mentioned earlier, but that might as well be infinity.
+omitting *n* results in an upper bound of infinity.
Readers of a reductionist bent may notice that the three other qualifiers can
all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}``
@@ -293,6 +289,8 @@ Putting REs in strings keeps the Python language simpler, but has one
disadvantage which is the topic of the next section.
+.. _the-backslash-plague:
+
The Backslash Plague
--------------------
@@ -331,6 +329,13 @@ backslashes are not handled in any special way in a string literal prefixed with
while ``"\n"`` is a one-character string containing a newline. Regular
expressions will often be written in Python code using this raw string notation.
+In addition, special escape sequences that are valid in regular expressions,
+but not valid as Python string literals, now result in a
+:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`,
+which means the sequences will be invalid if raw string notation or escaping
+the backslashes isn't used.
+
+
+-------------------+------------------+
| Regular String | Raw string |
+===================+==================+
@@ -366,7 +371,7 @@ for a complete listing.
| | returns them as an :term:`iterator`. |
+------------------+-----------------------------------------------+
-:meth:`~re.regex.match` and :meth:`~re.regex.search` return ``None`` if no match can be found. If
+:meth:`~re.pattern.match` and :meth:`~re.pattern.search` return ``None`` if no match can be found. If
they're successful, a :ref:`match object ` instance is returned,
containing information about the match: where it starts and ends, the substring
it matched, and more.
@@ -388,24 +393,24 @@ Python interpreter, import the :mod:`re` module, and compile a RE::
Now, you can try matching various strings against the RE ``[a-z]+``. An empty
string shouldn't match at all, since ``+`` means 'one or more repetitions'.
-:meth:`match` should return ``None`` in this case, which will cause the
+:meth:`~re.pattern.match` should return ``None`` in this case, which will cause the
interpreter to print no output. You can explicitly print the result of
-:meth:`match` to make this clear. ::
+:meth:`!match` to make this clear. ::
>>> p.match("")
>>> print(p.match(""))
None
Now, let's try it on a string that it should match, such as ``tempo``. In this
-case, :meth:`match` will return a :ref:`match object `, so you
+case, :meth:`~re.pattern.match` will return a :ref:`match object `, so you
should store the result in a variable for later use. ::
>>> m = p.match('tempo')
- >>> m #doctest: +ELLIPSIS
+ >>> m
<_sre.SRE_Match object; span=(0, 5), match='tempo'>
Now you can query the :ref:`match object ` for information
-about the matching string. :ref:`match object ` instances
+about the matching string. Match object instances
also have several methods and attributes; the most important ones are:
+------------------+--------------------------------------------+
@@ -432,15 +437,15 @@ Trying these methods will soon clarify their meaning::
:meth:`~re.match.group` returns the substring that was matched by the RE. :meth:`~re.match.start`
and :meth:`~re.match.end` return the starting and ending index of the match. :meth:`~re.match.span`
-returns both start and end indexes in a single tuple. Since the :meth:`match`
-method only checks if the RE matches at the start of a string, :meth:`start`
-will always be zero. However, the :meth:`search` method of patterns
+returns both start and end indexes in a single tuple. Since the :meth:`~re.pattern.match`
+method only checks if the RE matches at the start of a string, :meth:`!start`
+will always be zero. However, the :meth:`~re.pattern.search` method of patterns
scans through the string, so the match may not start at zero in that
case. ::
>>> print(p.match('::: message'))
None
- >>> m = p.search('::: message'); print(m) #doctest: +ELLIPSIS
+ >>> m = p.search('::: message'); print(m)
<_sre.SRE_Match object; span=(4, 11), match='message'>
>>> m.group()
'message'
@@ -459,14 +464,20 @@ In actual programs, the most common style is to store the
print('No match')
Two pattern methods return all of the matches for a pattern.
-:meth:`~re.regex.findall` returns a list of matching strings::
+:meth:`~re.pattern.findall` returns a list of matching strings::
- >>> p = re.compile('\d+')
+ >>> p = re.compile(r'\d+')
>>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping')
['12', '11', '10']
-:meth:`findall` has to create the entire list before it can be returned as the
-result. The :meth:`~re.regex.finditer` method returns a sequence of
+The ``r`` prefix, making the literal a raw string literal, is needed in this
+example because escape sequences in a normal "cooked" string literal that are
+not recognized by Python, as opposed to regular expressions, now result in a
+:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See
+:ref:`the-backslash-plague`.
+
+:meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the
+result. The :meth:`~re.Pattern.finditer` method returns a sequence of
:ref:`match object ` instances as an :term:`iterator`::
>>> iterator = p.finditer('12 drummers drumming, 11 ... 10 ...')
@@ -529,14 +540,14 @@ of each one.
| | characters with the respective property. |
+---------------------------------+--------------------------------------------+
| :const:`DOTALL`, :const:`S` | Make ``.`` match any character, including |
-| | newlines |
+| | newlines. |
+---------------------------------+--------------------------------------------+
-| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches |
+| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches. |
+---------------------------------+--------------------------------------------+
-| :const:`LOCALE`, :const:`L` | Do a locale-aware match |
+| :const:`LOCALE`, :const:`L` | Do a locale-aware match. |
+---------------------------------+--------------------------------------------+
| :const:`MULTILINE`, :const:`M` | Multi-line matching, affecting ``^`` and |
-| | ``$`` |
+| | ``$``. |
+---------------------------------+--------------------------------------------+
| :const:`VERBOSE`, :const:`X` | Enable verbose REs, which can be organized |
| (for 'extended') | more cleanly and understandably. |
@@ -549,27 +560,41 @@ of each one.
Perform case-insensitive matching; character class and literal strings will
match letters by ignoring case. For example, ``[A-Z]`` will match lowercase
- letters, too, and ``Spam`` will match ``Spam``, ``spam``, or ``spAM``. This
- lowercasing doesn't take the current locale into account; it will if you also
- set the :const:`LOCALE` flag.
+ letters, too. Full Unicode matching also works unless the :const:`ASCII`
+ flag is used to disable non-ASCII matches. When the Unicode patterns
+ ``[a-z]`` or ``[A-Z]`` are used in combination with the :const:`IGNORECASE`
+ flag, they will match the 52 ASCII letters and 4 additional non-ASCII
+ letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131,
+ Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and
+ 'K' (U+212A, Kelvin sign). ``Spam`` will match ``'Spam'``, ``'spam'``,
+ ``'spAM'``, or ``'ſpam'`` (the latter is matched only in Unicode mode).
+ This lowercasing doesn't take the current locale into account;
+ it will if you also set the :const:`LOCALE` flag.
.. data:: L
LOCALE
:noindex:
- Make ``\w``, ``\W``, ``\b``, and ``\B``, dependent on the current locale
- instead of the Unicode database.
-
- Locales are a feature of the C library intended to help in writing programs that
- take account of language differences. For example, if you're processing French
- text, you'd want to be able to write ``\w+`` to match words, but ``\w`` only
- matches the character class ``[A-Za-z]``; it won't match ``'é'`` or ``'ç'``. If
- your system is configured properly and a French locale is selected, certain C
- functions will tell the program that ``'é'`` should also be considered a letter.
+ Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching dependent
+ on the current locale instead of the Unicode database.
+
+ Locales are a feature of the C library intended to help in writing programs
+ that take account of language differences. For example, if you're
+ processing encoded French text, you'd want to be able to write ``\w+`` to
+ match words, but ``\w`` only matches the character class ``[A-Za-z]`` in
+ bytes patterns; it won't match bytes corresponding to ``é`` or ``ç``.
+ If your system is configured properly and a French locale is selected,
+ certain C functions will tell the program that the byte corresponding to
+ ``é`` should also be considered a letter.
Setting the :const:`LOCALE` flag when compiling a regular expression will cause
the resulting compiled object to use these C functions for ``\w``; this is
slower, but also enables ``\w+`` to match French words as you'd expect.
+ The use of this flag is discouraged in Python 3 as the locale mechanism
+ is very unreliable, it only handles one "culture" at a time, and it only
+ works with 8-bit locales. Unicode matching is already enabled by default
+ in Python 3 for Unicode (str) patterns, and it is able to handle different
+ locales/languages.
.. data:: M
@@ -667,11 +692,11 @@ zero-width assertions should never be repeated, because if they match once at a
given location, they can obviously be matched an infinite number of times.
``|``
- Alternation, or the "or" operator. If A and B are regular expressions,
- ``A|B`` will match any string that matches either ``A`` or ``B``. ``|`` has very
+ Alternation, or the "or" operator. If *A* and *B* are regular expressions,
+ ``A|B`` will match any string that matches either *A* or *B*. ``|`` has very
low precedence in order to make it work reasonably when you're alternating
- multi-character strings. ``Crow|Servo`` will match either ``Crow`` or ``Servo``,
- not ``Cro``, a ``'w'`` or an ``'S'``, and ``ervo``.
+ multi-character strings. ``Crow|Servo`` will match either ``'Crow'`` or ``'Servo'``,
+ not ``'Cro'``, a ``'w'`` or an ``'S'``, and ``'ervo'``.
To match a literal ``'|'``, use ``\|``, or enclose it inside a character class,
as in ``[|]``.
@@ -689,8 +714,7 @@ given location, they can obviously be matched an infinite number of times.
>>> print(re.search('^From', 'Reciting From Memory'))
None
- .. To match a literal \character{\^}, use \regexp{\e\^} or enclose it
- .. inside a character class, as in \regexp{[{\e}\^]}.
+ To match a literal ``'^'``, use ``\^``.
``$``
Matches at the end of a line, which is defined as either the end of the string,
@@ -725,7 +749,7 @@ given location, they can obviously be matched an infinite number of times.
match when it's contained inside another word. ::
>>> p = re.compile(r'\bclass\b')
- >>> print(p.search('no class at all')) #doctest: +ELLIPSIS
+ >>> print(p.search('no class at all'))
<_sre.SRE_Match object; span=(3, 8), match='class'>
>>> print(p.search('the declassified algorithm'))
None
@@ -743,7 +767,7 @@ given location, they can obviously be matched an infinite number of times.
>>> p = re.compile('\bclass\b')
>>> print(p.search('no class at all'))
None
- >>> print(p.search('\b' + 'class' + '\b')) #doctest: +ELLIPSIS
+ >>> print(p.search('\b' + 'class' + '\b'))
<_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'>
Second, inside a character class, where there's no use for this assertion,
@@ -762,7 +786,9 @@ Frequently you need to obtain more information than just whether the RE matched
or not. Regular expressions are often used to dissect strings by writing a RE
divided into several subgroups which match different components of interest.
For example, an RFC-822 header line is divided into a header name and a value,
-separated by a ``':'``, like this::
+separated by a ``':'``, like this:
+
+.. code-block:: none
From: author@example.com
User-Agent: Thunderbird 1.5.0.9 (X11/20061227)
@@ -786,7 +812,8 @@ of a group with a repeating qualifier, such as ``*``, ``+``, ``?``, or
Groups indicated with ``'('``, ``')'`` also capture the starting and ending
index of the text that they match; this can be retrieved by passing an argument
-to :meth:`group`, :meth:`start`, :meth:`end`, and :meth:`span`. Groups are
+to :meth:`~re.match.group`, :meth:`~re.match.start`, :meth:`~re.match.end`, and
+:meth:`~re.match.span`. Groups are
numbered starting with 0. Group 0 is always present; it's the whole RE, so
:ref:`match object ` methods all have group 0 as their default
argument. Later we'll see how to express groups that don't capture the span
@@ -812,13 +839,13 @@ from left to right. ::
>>> m.group(2)
'b'
-:meth:`group` can be passed multiple group numbers at a time, in which case it
+:meth:`~re.match.group` can be passed multiple group numbers at a time, in which case it
will return a tuple containing the corresponding values for those groups. ::
>>> m.group(2,1,2)
('b', 'abc', 'b')
-The :meth:`groups` method returns a tuple containing the strings for all the
+The :meth:`~re.match.groups` method returns a tuple containing the strings for all the
subgroups, from 1 up to however many there are. ::
>>> m.groups()
@@ -834,7 +861,7 @@ backreferences in a RE.
For example, the following RE detects doubled words in a string. ::
- >>> p = re.compile(r'(\b\w+)\s+\1')
+ >>> p = re.compile(r'\b(\w+)\s+\1\b')
>>> p.search('Paris in the the spring').group()
'the the'
@@ -933,9 +960,9 @@ number of the group. There's naturally a variant that uses the group name
instead of the number. This is another Python extension: ``(?P=name)`` indicates
that the contents of the group called *name* should again be matched at the
current point. The regular expression for finding doubled words,
-``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``::
+``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``::
- >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)')
+ >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b')
>>> p.search('Paris in the the spring').group()
'the the'
@@ -1034,7 +1061,7 @@ using the following pattern methods:
| ``sub()`` | Find all substrings where the RE matches, and |
| | replace them with a different string |
+------------------+-----------------------------------------------+
-| ``subn()`` | Does the same thing as :meth:`sub`, but |
+| ``subn()`` | Does the same thing as :meth:`!sub`, but |
| | returns the new string and the number of |
| | replacements |
+------------------+-----------------------------------------------+
@@ -1043,10 +1070,10 @@ using the following pattern methods:
Splitting Strings
-----------------
-The :meth:`split` method of a pattern splits a string apart
+The :meth:`~re.pattern.split` method of a pattern splits a string apart
wherever the RE matches, returning a list of the pieces. It's similar to the
-:meth:`split` method of strings but provides much more generality in the
-delimiters that you can split by; string :meth:`split` only supports splitting by
+:meth:`~str.split` method of strings but provides much more generality in the
+delimiters that you can split by; string :meth:`!split` only supports splitting by
whitespace or by a fixed string. As you'd expect, there's a module-level
:func:`re.split` function, too.
@@ -1086,11 +1113,11 @@ following calls::
The module-level function :func:`re.split` adds the RE to be used as the first
argument, but is otherwise the same. ::
- >>> re.split('[\W]+', 'Words, words, words.')
+ >>> re.split(r'[\W]+', 'Words, words, words.')
['Words', 'words', 'words', '']
- >>> re.split('([\W]+)', 'Words, words, words.')
+ >>> re.split(r'([\W]+)', 'Words, words, words.')
['Words', ', ', 'words', ', ', 'words', '.', '']
- >>> re.split('[\W]+', 'Words, words, words.', 1)
+ >>> re.split(r'[\W]+', 'Words, words, words.', 1)
['Words', 'words, words.']
@@ -1098,7 +1125,7 @@ Search and Replace
------------------
Another common task is to find all the matches for a pattern, and replace them
-with a different string. The :meth:`sub` method takes a replacement value,
+with a different string. The :meth:`~re.pattern.sub` method takes a replacement value,
which can be either a string or a function, and the string to be processed.
.. method:: .sub(replacement, string[, count=0])
@@ -1112,7 +1139,7 @@ which can be either a string or a function, and the string to be processed.
replaced; *count* must be a non-negative integer. The default value of 0 means
to replace all occurrences.
-Here's a simple example of using the :meth:`sub` method. It replaces colour
+Here's a simple example of using the :meth:`~re.pattern.sub` method. It replaces colour
names with the word ``colour``::
>>> p = re.compile('(blue|white|red)')
@@ -1121,7 +1148,7 @@ names with the word ``colour``::
>>> p.sub('colour', 'blue socks and red shoes', count=1)
'colour socks and red shoes'
-The :meth:`subn` method does the same work, but returns a 2-tuple containing the
+The :meth:`~re.pattern.subn` method does the same work, but returns a 2-tuple containing the
new string value and the number of replacements that were performed::
>>> p = re.compile('(blue|white|red)')
@@ -1206,24 +1233,24 @@ Use String Methods
Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed
string, or a single character class, and you're not using any :mod:`re` features
-such as the :const:`IGNORECASE` flag, then the full power of regular expressions
+such as the :const:`~re.IGNORECASE` flag, then the full power of regular expressions
may not be required. Strings have several methods for performing operations with
fixed strings and they're usually much faster, because the implementation is a
single small C loop that's been optimized for the purpose, instead of the large,
more generalized regular expression engine.
One example might be replacing a single fixed string with another one; for
-example, you might replace ``word`` with ``deed``. ``re.sub()`` seems like the
-function to use for this, but consider the :meth:`replace` method. Note that
-:func:`replace` will also replace ``word`` inside words, turning ``swordfish``
+example, you might replace ``word`` with ``deed``. :func:`re.sub` seems like the
+function to use for this, but consider the :meth:`~str.replace` method. Note that
+:meth:`!replace` will also replace ``word`` inside words, turning ``swordfish``
into ``sdeedfish``, but the naive RE ``word`` would have done that, too. (To
avoid performing the substitution on parts of words, the pattern would have to
be ``\bword\b``, in order to require that ``word`` have a word boundary on
-either side. This takes the job beyond :meth:`replace`'s abilities.)
+either side. This takes the job beyond :meth:`!replace`'s abilities.)
Another common task is deleting every occurrence of a single character from a
string or replacing it with another single character. You might do this with
-something like ``re.sub('\n', ' ', S)``, but :meth:`translate` is capable of
+something like ``re.sub('\n', ' ', S)``, but :meth:`~str.translate` is capable of
doing both tasks and will be faster than any regular expression operation can
be.
@@ -1234,18 +1261,18 @@ can be solved with a faster and simpler string method.
match() versus search()
-----------------------
-The :func:`match` function only checks if the RE matches at the beginning of the
-string while :func:`search` will scan forward through the string for a match.
-It's important to keep this distinction in mind. Remember, :func:`match` will
+The :func:`~re.match` function only checks if the RE matches at the beginning of the
+string while :func:`~re.search` will scan forward through the string for a match.
+It's important to keep this distinction in mind. Remember, :func:`!match` will
only report a successful match which will start at 0; if the match wouldn't
-start at zero, :func:`match` will *not* report it. ::
+start at zero, :func:`!match` will *not* report it. ::
>>> print(re.match('super', 'superstition').span())
(0, 5)
>>> print(re.match('super', 'insuperable'))
None
-On the other hand, :func:`search` will scan forward through the string,
+On the other hand, :func:`~re.search` will scan forward through the string,
reporting the first match it finds. ::
>>> print(re.search('super', 'superstition').span())
@@ -1284,12 +1311,12 @@ doesn't work because of the greedy nature of ``.*``. ::
>>> print(re.match('<.*>', s).group())
Title
-The RE matches the ``'<'`` in ````, and the ``.*`` consumes the rest of
+The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of
the string. There's still more left in the RE, though, and the ``>`` can't
match at the end of the string, so the regular expression engine has to
backtrack character by character until it finds a match for the ``>``. The
-final match extends from the ``'<'`` in ```` to the ``'>'`` in
-````, which isn't what you want.
+final match extends from the ``'<'`` in ``''`` to the ``'>'`` in
+``''``, which isn't what you want.
In this case, the solution is to use the non-greedy qualifiers ``*?``, ``+?``,
``??``, or ``{m,n}?``, which match as *little* text as possible. In the above
@@ -1315,7 +1342,7 @@ notation, but they're not terribly readable. REs of moderate complexity can
become lengthy collections of backslashes, parentheses, and metacharacters,
making them difficult to read and understand.
-For such REs, specifying the ``re.VERBOSE`` flag when compiling the regular
+For such REs, specifying the :const:`re.VERBOSE` flag when compiling the regular
expression can be helpful, because it allows you to format the regular
expression more clearly.
@@ -1354,5 +1381,5 @@ Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately,
it exclusively concentrates on Perl and Java's flavours of regular expressions,
and doesn't contain any Python material at all, so it won't be useful as a
reference for programming in Python. (The first edition covered Python's
-now-removed :mod:`regex` module, which won't help you much.) Consider checking
+now-removed :mod:`!regex` module, which won't help you much.) Consider checking
it out from your library.
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index a48ae1f5faba7e2..ec5d48e64314845 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -30,7 +30,9 @@ spellings such as 'coöperate'.)
For a while people just wrote programs that didn't display accents.
In the mid-1980s an Apple II BASIC program written by a French speaker
-might have lines like these::
+might have lines like these:
+
+.. code-block:: basic
PRINT "MISE A JOUR TERMINEE"
PRINT "PARAMETRES ENREGISTRES"
@@ -43,9 +45,9 @@ hold values ranging from 0 to 255. ASCII codes only went up to 127, so some
machines assigned values between 128 and 255 to accented characters. Different
machines had different codes, however, which led to problems exchanging files.
Eventually various commonly used sets of values for the 128--255 range emerged.
-Some were true standards, defined by the International Standards Organization,
-and some were *de facto* conventions that were invented by one company or
-another and managed to catch on.
+Some were true standards, defined by the International Organization for
+Standardization, and some were *de facto* conventions that were invented by one
+company or another and managed to catch on.
255 characters aren't very many. For example, you can't fit both the accented
characters used in Western Europe and the Cyrillic alphabet used for Russian
@@ -463,7 +465,7 @@ The string in this example has the number 57 written in both Thai and
Arabic numerals::
import re
- p = re.compile('\d+')
+ p = re.compile(r'\d+')
s = "Over \u0e55\u0e57 57 flavours"
m = p.search(s)
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index 18b5c6556be1c96..7505e7564422abc 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -34,8 +34,8 @@ handling common situations - like basic authentication, cookies, proxies and so
on. These are provided by objects called handlers and openers.
urllib.request supports fetching URLs for many "URL schemes" (identified by the string
-before the ":" in URL - for example "ftp" is the URL scheme of
-"ftp://python.org/") using their associated network protocols (e.g. FTP, HTTP).
+before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of
+``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP).
This tutorial focuses on the most common case, HTTP.
For straightforward situations *urlopen* is very easy to use. But as soon as you
@@ -56,12 +56,20 @@ The simplest way to use urllib.request is as follows::
with urllib.request.urlopen('http://python.org/') as response:
html = response.read()
-If you wish to retrieve a resource via URL and store it in a temporary location,
-you can do so via the :func:`~urllib.request.urlretrieve` function::
+If you wish to retrieve a resource via URL and store it in a temporary
+location, you can do so via the :func:`shutil.copyfileobj` and
+:func:`tempfile.NamedTemporaryFile` functions::
+ import shutil
+ import tempfile
import urllib.request
- local_filename, headers = urllib.request.urlretrieve('http://python.org/')
- html = open(local_filename)
+
+ with urllib.request.urlopen('http://python.org/') as response:
+ with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+ shutil.copyfileobj(response, tmp_file)
+
+ with open(tmp_file.name) as html:
+ pass
Many uses of urllib will be that simple (note that instead of an 'http:' URL we
could have used a URL starting with 'ftp:', 'file:', etc.). However, it's the
@@ -231,7 +239,7 @@ a different URL, urllib will handle that for you). For those it can't handle,
urlopen will raise an :exc:`HTTPError`. Typical errors include '404' (page not
found), '403' (request forbidden), and '401' (authentication required).
-See section 10 of RFC 2616 for a reference on all the HTTP error codes.
+See section 10 of :rfc:`2616` for a reference on all the HTTP error codes.
The :exc:`HTTPError` instance raised will have an integer 'code' attribute, which
corresponds to the error sent by the server.
@@ -244,7 +252,7 @@ codes in the 100--299 range indicate success, you will usually only see error
codes in the 400--599 range.
:attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of
-response codes in that shows all the response codes used by RFC 2616. The
+response codes in that shows all the response codes used by :rfc:`2616`. The
dictionary is reproduced here for convenience ::
# Table mapping response codes to messages; entries have the
@@ -457,7 +465,9 @@ error code) requesting authentication. This specifies the authentication scheme
and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME
realm="REALM"``.
-e.g. ::
+e.g.
+
+.. code-block:: none
WWW-Authenticate: Basic realm="cPanel Users"
@@ -511,10 +521,10 @@ than the URL you pass to .add_password() will also match. ::
``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme
component and the hostname and optionally the port number)
-e.g. "http://example.com/" *or* an "authority" (i.e. the hostname,
-optionally including the port number) e.g. "example.com" or "example.com:8080"
+e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname,
+optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"``
(the latter example includes a port number). The authority, if present, must
-NOT contain the "userinfo" component - for example "joe:password@example.com" is
+NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is
not correct.
diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c
new file mode 100644
index 000000000000000..fb2c7b2a430e644
--- /dev/null
+++ b/Doc/includes/custom.c
@@ -0,0 +1,39 @@
+#include
+
+typedef struct {
+ PyObject_HEAD
+ /* Type-specific fields go here. */
+} CustomObject;
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT,
+ .tp_new = PyType_GenericNew,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c
new file mode 100644
index 000000000000000..51ab4b80d680980
--- /dev/null
+++ b/Doc/includes/custom2.c
@@ -0,0 +1,132 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_XDECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_XDECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0,
+ "first name"},
+ {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0,
+ "last name"},
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ if (self->first == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "first");
+ return NULL;
+ }
+ if (self->last == NULL) {
+ PyErr_SetString(PyExc_AttributeError, "last");
+ return NULL;
+ }
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom2.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom2",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom2(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c
new file mode 100644
index 000000000000000..09e87355b91afa2
--- /dev/null
+++ b/Doc/includes/custom3.c
@@ -0,0 +1,183 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ Py_XDECREF(self->first);
+ Py_XDECREF(self->last);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_getfirst(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->first);
+ return self->first;
+}
+
+static int
+Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+{
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ tmp = self->first;
+ Py_INCREF(value);
+ self->first = value;
+ Py_DECREF(tmp);
+ return 0;
+}
+
+static PyObject *
+Custom_getlast(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->last);
+ return self->last;
+}
+
+static int
+Custom_setlast(CustomObject *self, PyObject *value, void *closure)
+{
+ PyObject *tmp;
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+ tmp = self->last;
+ Py_INCREF(value);
+ self->last = value;
+ Py_DECREF(tmp);
+ return 0;
+}
+
+static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom3.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+ .tp_getset = Custom_getsetters,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom3",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom3(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c
new file mode 100644
index 000000000000000..0994d8fda0e51fa
--- /dev/null
+++ b/Doc/includes/custom4.c
@@ -0,0 +1,197 @@
+#include
+#include "structmember.h"
+
+typedef struct {
+ PyObject_HEAD
+ PyObject *first; /* first name */
+ PyObject *last; /* last name */
+ int number;
+} CustomObject;
+
+static int
+Custom_traverse(CustomObject *self, visitproc visit, void *arg)
+{
+ Py_VISIT(self->first);
+ Py_VISIT(self->last);
+ return 0;
+}
+
+static int
+Custom_clear(CustomObject *self)
+{
+ Py_CLEAR(self->first);
+ Py_CLEAR(self->last);
+ return 0;
+}
+
+static void
+Custom_dealloc(CustomObject *self)
+{
+ PyObject_GC_UnTrack(self);
+ Custom_clear(self);
+ Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+ CustomObject *self;
+ self = (CustomObject *) type->tp_alloc(type, 0);
+ if (self != NULL) {
+ self->first = PyUnicode_FromString("");
+ if (self->first == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->last = PyUnicode_FromString("");
+ if (self->last == NULL) {
+ Py_DECREF(self);
+ return NULL;
+ }
+ self->number = 0;
+ }
+ return (PyObject *) self;
+}
+
+static int
+Custom_init(CustomObject *self, PyObject *args, PyObject *kwds)
+{
+ static char *kwlist[] = {"first", "last", "number", NULL};
+ PyObject *first = NULL, *last = NULL, *tmp;
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist,
+ &first, &last,
+ &self->number))
+ return -1;
+
+ if (first) {
+ tmp = self->first;
+ Py_INCREF(first);
+ self->first = first;
+ Py_DECREF(tmp);
+ }
+ if (last) {
+ tmp = self->last;
+ Py_INCREF(last);
+ self->last = last;
+ Py_DECREF(tmp);
+ }
+ return 0;
+}
+
+static PyMemberDef Custom_members[] = {
+ {"number", T_INT, offsetof(CustomObject, number), 0,
+ "custom number"},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_getfirst(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->first);
+ return self->first;
+}
+
+static int
+Custom_setfirst(CustomObject *self, PyObject *value, void *closure)
+{
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The first attribute value must be a string");
+ return -1;
+ }
+ Py_INCREF(value);
+ Py_CLEAR(self->first);
+ self->first = value;
+ return 0;
+}
+
+static PyObject *
+Custom_getlast(CustomObject *self, void *closure)
+{
+ Py_INCREF(self->last);
+ return self->last;
+}
+
+static int
+Custom_setlast(CustomObject *self, PyObject *value, void *closure)
+{
+ if (value == NULL) {
+ PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) {
+ PyErr_SetString(PyExc_TypeError,
+ "The last attribute value must be a string");
+ return -1;
+ }
+ Py_INCREF(value);
+ Py_CLEAR(self->last);
+ self->last = value;
+ return 0;
+}
+
+static PyGetSetDef Custom_getsetters[] = {
+ {"first", (getter) Custom_getfirst, (setter) Custom_setfirst,
+ "first name", NULL},
+ {"last", (getter) Custom_getlast, (setter) Custom_setlast,
+ "last name", NULL},
+ {NULL} /* Sentinel */
+};
+
+static PyObject *
+Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return PyUnicode_FromFormat("%S %S", self->first, self->last);
+}
+
+static PyMethodDef Custom_methods[] = {
+ {"name", (PyCFunction) Custom_name, METH_NOARGS,
+ "Return the name, combining the first and last name"
+ },
+ {NULL} /* Sentinel */
+};
+
+static PyTypeObject CustomType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "custom4.Custom",
+ .tp_doc = "Custom objects",
+ .tp_basicsize = sizeof(CustomObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
+ .tp_new = Custom_new,
+ .tp_init = (initproc) Custom_init,
+ .tp_dealloc = (destructor) Custom_dealloc,
+ .tp_traverse = (traverseproc) Custom_traverse,
+ .tp_clear = (inquiry) Custom_clear,
+ .tp_members = Custom_members,
+ .tp_methods = Custom_methods,
+ .tp_getset = Custom_getsetters,
+};
+
+static PyModuleDef custommodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "custom4",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_custom4(void)
+{
+ PyObject *m;
+ if (PyType_Ready(&CustomType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&custommodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&CustomType);
+ PyModule_AddObject(m, "Custom", (PyObject *) &CustomType);
+ return m;
+}
diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py
index 2e142b1e3b71e4b..df7ca6f3faa332c 100644
--- a/Doc/includes/email-alternative.py
+++ b/Doc/includes/email-alternative.py
@@ -32,7 +32,7 @@
Salut!
Cela ressemble à un excellent
-
recipie
déjeuner.
diff --git a/Doc/includes/email-read-alternative.py b/Doc/includes/email-read-alternative.py
index 3f5ab24c0fbdd25..5ea84e62584a465 100644
--- a/Doc/includes/email-read-alternative.py
+++ b/Doc/includes/email-read-alternative.py
@@ -21,7 +21,7 @@
print('From:', msg['from'])
print('Subject:', msg['subject'])
-# If we want to print a priview of the message content, we can extract whatever
+# If we want to print a preview of the message content, we can extract whatever
# the least formatted payload is and print the first three lines. Of course,
# if the message has no plain text part printing the first three lines of html
# is probably useless, but this is just a conceptual example.
diff --git a/Doc/includes/noddy.c b/Doc/includes/noddy.c
deleted file mode 100644
index 19a27a89e883b3b..000000000000000
--- a/Doc/includes/noddy.c
+++ /dev/null
@@ -1,56 +0,0 @@
-#include
-
-typedef struct {
- PyObject_HEAD
- /* Type-specific fields go here. */
-} noddy_NoddyObject;
-
-static PyTypeObject noddy_NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(noddy_NoddyObject), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT, /* tp_flags */
- "Noddy objects", /* tp_doc */
-};
-
-static PyModuleDef noddymodule = {
- PyModuleDef_HEAD_INIT,
- "noddy",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy(void)
-{
- PyObject* m;
-
- noddy_NoddyType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&noddy_NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&noddy_NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy2.c b/Doc/includes/noddy2.c
deleted file mode 100644
index 964155845fee83e..000000000000000
--- a/Doc/includes/noddy2.c
+++ /dev/null
@@ -1,172 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first; /* first name */
- PyObject *last; /* last name */
- int number;
-} Noddy;
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
-}
-
-
-static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- "Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy2module = {
- PyModuleDef_HEAD_INIT,
- "noddy2",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy2(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy2module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy3.c b/Doc/includes/noddy3.c
deleted file mode 100644
index 8a5a753ca439eb9..000000000000000
--- a/Doc/includes/noddy3.c
+++ /dev/null
@@ -1,225 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
-} Noddy;
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- Py_XDECREF(self->first);
- Py_XDECREF(self->last);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_DECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_DECREF(tmp);
- }
-
- return 0;
-}
-
-static PyMemberDef Noddy_members[] = {
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_getfirst(Noddy *self, void *closure)
-{
- Py_INCREF(self->first);
- return self->first;
-}
-
-static int
-Noddy_setfirst(Noddy *self, PyObject *value, void *closure)
-{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The first attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->first);
- Py_INCREF(value);
- self->first = value;
-
- return 0;
-}
-
-static PyObject *
-Noddy_getlast(Noddy *self, void *closure)
-{
- Py_INCREF(self->last);
- return self->last;
-}
-
-static int
-Noddy_setlast(Noddy *self, PyObject *value, void *closure)
-{
- if (value == NULL) {
- PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute");
- return -1;
- }
-
- if (! PyUnicode_Check(value)) {
- PyErr_SetString(PyExc_TypeError,
- "The last attribute value must be a string");
- return -1;
- }
-
- Py_DECREF(self->last);
- Py_INCREF(value);
- self->last = value;
-
- return 0;
-}
-
-static PyGetSetDef Noddy_getseters[] = {
- {"first",
- (getter)Noddy_getfirst, (setter)Noddy_setfirst,
- "first name",
- NULL},
- {"last",
- (getter)Noddy_getlast, (setter)Noddy_setlast,
- "last name",
- NULL},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- "Noddy objects", /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- Noddy_getseters, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy3module = {
- PyModuleDef_HEAD_INIT,
- "noddy3",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy3(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy3module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c
deleted file mode 100644
index eb9622a87d99cc9..000000000000000
--- a/Doc/includes/noddy4.c
+++ /dev/null
@@ -1,207 +0,0 @@
-#include
-#include "structmember.h"
-
-typedef struct {
- PyObject_HEAD
- PyObject *first;
- PyObject *last;
- int number;
-} Noddy;
-
-static int
-Noddy_traverse(Noddy *self, visitproc visit, void *arg)
-{
- int vret;
-
- if (self->first) {
- vret = visit(self->first, arg);
- if (vret != 0)
- return vret;
- }
- if (self->last) {
- vret = visit(self->last, arg);
- if (vret != 0)
- return vret;
- }
-
- return 0;
-}
-
-static int
-Noddy_clear(Noddy *self)
-{
- PyObject *tmp;
-
- tmp = self->first;
- self->first = NULL;
- Py_XDECREF(tmp);
-
- tmp = self->last;
- self->last = NULL;
- Py_XDECREF(tmp);
-
- return 0;
-}
-
-static void
-Noddy_dealloc(Noddy* self)
-{
- Noddy_clear(self);
- Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject *
-Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-{
- Noddy *self;
-
- self = (Noddy *)type->tp_alloc(type, 0);
- if (self != NULL) {
- self->first = PyUnicode_FromString("");
- if (self->first == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->last = PyUnicode_FromString("");
- if (self->last == NULL) {
- Py_DECREF(self);
- return NULL;
- }
-
- self->number = 0;
- }
-
- return (PyObject *)self;
-}
-
-static int
-Noddy_init(Noddy *self, PyObject *args, PyObject *kwds)
-{
- PyObject *first=NULL, *last=NULL, *tmp;
-
- static char *kwlist[] = {"first", "last", "number", NULL};
-
- if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist,
- &first, &last,
- &self->number))
- return -1;
-
- if (first) {
- tmp = self->first;
- Py_INCREF(first);
- self->first = first;
- Py_XDECREF(tmp);
- }
-
- if (last) {
- tmp = self->last;
- Py_INCREF(last);
- self->last = last;
- Py_XDECREF(tmp);
- }
-
- return 0;
-}
-
-
-static PyMemberDef Noddy_members[] = {
- {"first", T_OBJECT_EX, offsetof(Noddy, first), 0,
- "first name"},
- {"last", T_OBJECT_EX, offsetof(Noddy, last), 0,
- "last name"},
- {"number", T_INT, offsetof(Noddy, number), 0,
- "noddy number"},
- {NULL} /* Sentinel */
-};
-
-static PyObject *
-Noddy_name(Noddy* self)
-{
- if (self->first == NULL) {
- PyErr_SetString(PyExc_AttributeError, "first");
- return NULL;
- }
-
- if (self->last == NULL) {
- PyErr_SetString(PyExc_AttributeError, "last");
- return NULL;
- }
-
- return PyUnicode_FromFormat("%S %S", self->first, self->last);
-}
-
-static PyMethodDef Noddy_methods[] = {
- {"name", (PyCFunction)Noddy_name, METH_NOARGS,
- "Return the name, combining the first and last name"
- },
- {NULL} /* Sentinel */
-};
-
-static PyTypeObject NoddyType = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "noddy.Noddy", /* tp_name */
- sizeof(Noddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- (destructor)Noddy_dealloc, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE |
- Py_TPFLAGS_HAVE_GC, /* tp_flags */
- "Noddy objects", /* tp_doc */
- (traverseproc)Noddy_traverse, /* tp_traverse */
- (inquiry)Noddy_clear, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Noddy_methods, /* tp_methods */
- Noddy_members, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Noddy_init, /* tp_init */
- 0, /* tp_alloc */
- Noddy_new, /* tp_new */
-};
-
-static PyModuleDef noddy4module = {
- PyModuleDef_HEAD_INIT,
- "noddy4",
- "Example module that creates an extension type.",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_noddy4(void)
-{
- PyObject* m;
-
- if (PyType_Ready(&NoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&noddy4module);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&NoddyType);
- PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType);
- return m;
-}
diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c
index ead7bdd23209a3d..9caf1fdb20104af 100644
--- a/Doc/includes/run-func.c
+++ b/Doc/includes/run-func.c
@@ -3,7 +3,7 @@
int
main(int argc, char *argv[])
{
- PyObject *pName, *pModule, *pDict, *pFunc;
+ PyObject *pName, *pModule, *pFunc;
PyObject *pArgs, *pValue;
int i;
diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py
index b853d23b170985a..a38a39de3e7c86c 100644
--- a/Doc/includes/setup.py
+++ b/Doc/includes/setup.py
@@ -5,4 +5,5 @@
Extension("noddy2", ["noddy2.c"]),
Extension("noddy3", ["noddy3.c"]),
Extension("noddy4", ["noddy4.c"]),
+ Extension("shoddy", ["shoddy.c"]),
])
diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c
deleted file mode 100644
index 07a272124ceabaa..000000000000000
--- a/Doc/includes/shoddy.c
+++ /dev/null
@@ -1,99 +0,0 @@
-#include
-
-typedef struct {
- PyListObject list;
- int state;
-} Shoddy;
-
-
-static PyObject *
-Shoddy_increment(Shoddy *self, PyObject *unused)
-{
- self->state++;
- return PyLong_FromLong(self->state);
-}
-
-
-static PyMethodDef Shoddy_methods[] = {
- {"increment", (PyCFunction)Shoddy_increment, METH_NOARGS,
- PyDoc_STR("increment state counter")},
- {NULL, NULL},
-};
-
-static int
-Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds)
-{
- if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0)
- return -1;
- self->state = 0;
- return 0;
-}
-
-
-static PyTypeObject ShoddyType = {
- PyObject_HEAD_INIT(NULL)
- "shoddy.Shoddy", /* tp_name */
- sizeof(Shoddy), /* tp_basicsize */
- 0, /* tp_itemsize */
- 0, /* tp_dealloc */
- 0, /* tp_print */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_reserved */
- 0, /* tp_repr */
- 0, /* tp_as_number */
- 0, /* tp_as_sequence */
- 0, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- 0, /* tp_getattro */
- 0, /* tp_setattro */
- 0, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT |
- Py_TPFLAGS_BASETYPE, /* tp_flags */
- 0, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- 0, /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- Shoddy_methods, /* tp_methods */
- 0, /* tp_members */
- 0, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- (initproc)Shoddy_init, /* tp_init */
- 0, /* tp_alloc */
- 0, /* tp_new */
-};
-
-static PyModuleDef shoddymodule = {
- PyModuleDef_HEAD_INIT,
- "shoddy",
- "Shoddy module",
- -1,
- NULL, NULL, NULL, NULL, NULL
-};
-
-PyMODINIT_FUNC
-PyInit_shoddy(void)
-{
- PyObject *m;
-
- ShoddyType.tp_base = &PyList_Type;
- if (PyType_Ready(&ShoddyType) < 0)
- return NULL;
-
- m = PyModule_Create(&shoddymodule);
- if (m == NULL)
- return NULL;
-
- Py_INCREF(&ShoddyType);
- PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType);
- return m;
-}
diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py
index 015aa0de8c46d4d..b997c70668ace41 100644
--- a/Doc/includes/sqlite3/load_extension.py
+++ b/Doc/includes/sqlite3/load_extension.py
@@ -11,7 +11,7 @@
# alternatively you can load the extension using an API call:
# con.load_extension("./fts3.so")
-# disable extension laoding again
+# disable extension loading again
con.enable_load_extension(False)
# example from SQLite wiki
diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c
new file mode 100644
index 000000000000000..376dddfac09c685
--- /dev/null
+++ b/Doc/includes/sublist.c
@@ -0,0 +1,63 @@
+#include
+
+typedef struct {
+ PyListObject list;
+ int state;
+} SubListObject;
+
+static PyObject *
+SubList_increment(SubListObject *self, PyObject *unused)
+{
+ self->state++;
+ return PyLong_FromLong(self->state);
+}
+
+static PyMethodDef SubList_methods[] = {
+ {"increment", (PyCFunction) SubList_increment, METH_NOARGS,
+ PyDoc_STR("increment state counter")},
+ {NULL},
+};
+
+static int
+SubList_init(SubListObject *self, PyObject *args, PyObject *kwds)
+{
+ if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0)
+ return -1;
+ self->state = 0;
+ return 0;
+}
+
+static PyTypeObject SubListType = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "sublist.SubList",
+ .tp_doc = "SubList objects",
+ .tp_basicsize = sizeof(SubListObject),
+ .tp_itemsize = 0,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .tp_init = (initproc) SubList_init,
+ .tp_methods = SubList_methods,
+};
+
+static PyModuleDef sublistmodule = {
+ PyModuleDef_HEAD_INIT,
+ .m_name = "sublist",
+ .m_doc = "Example module that creates an extension type.",
+ .m_size = -1,
+};
+
+PyMODINIT_FUNC
+PyInit_sublist(void)
+{
+ PyObject *m;
+ SubListType.tp_base = &PyList_Type;
+ if (PyType_Ready(&SubListType) < 0)
+ return NULL;
+
+ m = PyModule_Create(&sublistmodule);
+ if (m == NULL)
+ return NULL;
+
+ Py_INCREF(&SubListType);
+ PyModule_AddObject(m, "SubList", (PyObject *) &SubListType);
+ return m;
+}
diff --git a/Doc/includes/test.py b/Doc/includes/test.py
index 9e9d4a67121cae1..09ebe3fec0bdbe3 100644
--- a/Doc/includes/test.py
+++ b/Doc/includes/test.py
@@ -1,181 +1,168 @@
-"""Test module for the noddy examples
+"""Test module for the custom examples
-Noddy 1:
+Custom 1:
->>> import noddy
->>> n1 = noddy.Noddy()
->>> n2 = noddy.Noddy()
->>> del n1
->>> del n2
+>>> import custom
+>>> c1 = custom.Custom()
+>>> c2 = custom.Custom()
+>>> del c1
+>>> del c2
-Noddy 2
+Custom 2
->>> import noddy2
->>> n1 = noddy2.Noddy('jim', 'fulton', 42)
->>> n1.first
+>>> import custom2
+>>> c1 = custom2.Custom('jim', 'fulton', 42)
+>>> c1.first
'jim'
->>> n1.last
+>>> c1.last
'fulton'
->>> n1.number
+>>> c1.number
42
->>> n1.name()
+>>> c1.name()
'jim fulton'
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n1.last = 'tell'
->>> n1.name()
+>>> c1.last = 'tell'
+>>> c1.name()
'will tell'
->>> del n1.first
->>> n1.name()
+>>> del c1.first
+>>> c1.name()
Traceback (most recent call last):
...
AttributeError: first
->>> n1.first
+>>> c1.first
Traceback (most recent call last):
...
AttributeError: first
->>> n1.first = 'drew'
->>> n1.first
+>>> c1.first = 'drew'
+>>> c1.first
'drew'
->>> del n1.number
+>>> del c1.number
Traceback (most recent call last):
...
TypeError: can't delete numeric/char attribute
->>> n1.number=2
->>> n1.number
+>>> c1.number=2
+>>> c1.number
2
->>> n1.first = 42
->>> n1.name()
+>>> c1.first = 42
+>>> c1.name()
'42 tell'
->>> n2 = noddy2.Noddy()
->>> n2.name()
+>>> c2 = custom2.Custom()
+>>> c2.name()
' '
->>> n2.first
+>>> c2.first
''
->>> n2.last
+>>> c2.last
''
->>> del n2.first
->>> n2.first
+>>> del c2.first
+>>> c2.first
Traceback (most recent call last):
...
AttributeError: first
->>> n2.first
+>>> c2.first
Traceback (most recent call last):
...
AttributeError: first
->>> n2.name()
+>>> c2.name()
Traceback (most recent call last):
File "", line 1, in ?
AttributeError: first
->>> n2.number
+>>> c2.number
0
->>> n3 = noddy2.Noddy('jim', 'fulton', 'waaa')
+>>> n3 = custom2.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
File "", line 1, in ?
-TypeError: an integer is required
->>> del n1
->>> del n2
+TypeError: an integer is required (got type str)
+>>> del c1
+>>> del c2
-Noddy 3
+Custom 3
->>> import noddy3
->>> n1 = noddy3.Noddy('jim', 'fulton', 42)
->>> n1 = noddy3.Noddy('jim', 'fulton', 42)
->>> n1.name()
+>>> import custom3
+>>> c1 = custom3.Custom('jim', 'fulton', 42)
+>>> c1 = custom3.Custom('jim', 'fulton', 42)
+>>> c1.name()
'jim fulton'
->>> del n1.first
+>>> del c1.first
Traceback (most recent call last):
File "", line 1, in ?
TypeError: Cannot delete the first attribute
->>> n1.first = 42
+>>> c1.first = 42
Traceback (most recent call last):
File "", line 1, in ?
TypeError: The first attribute value must be a string
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n2 = noddy3.Noddy()
->>> n2 = noddy3.Noddy()
->>> n2 = noddy3.Noddy()
->>> n3 = noddy3.Noddy('jim', 'fulton', 'waaa')
+>>> c2 = custom3.Custom()
+>>> c2 = custom3.Custom()
+>>> c2 = custom3.Custom()
+>>> n3 = custom3.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
File "", line 1, in ?
-TypeError: an integer is required
->>> del n1
->>> del n2
+TypeError: an integer is required (got type str)
+>>> del c1
+>>> del c2
-Noddy 4
+Custom 4
->>> import noddy4
->>> n1 = noddy4.Noddy('jim', 'fulton', 42)
->>> n1.first
+>>> import custom4
+>>> c1 = custom4.Custom('jim', 'fulton', 42)
+>>> c1.first
'jim'
->>> n1.last
+>>> c1.last
'fulton'
->>> n1.number
+>>> c1.number
42
->>> n1.name()
+>>> c1.name()
'jim fulton'
->>> n1.first = 'will'
->>> n1.name()
+>>> c1.first = 'will'
+>>> c1.name()
'will fulton'
->>> n1.last = 'tell'
->>> n1.name()
+>>> c1.last = 'tell'
+>>> c1.name()
'will tell'
->>> del n1.first
->>> n1.name()
+>>> del c1.first
Traceback (most recent call last):
...
-AttributeError: first
->>> n1.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n1.first = 'drew'
->>> n1.first
+TypeError: Cannot delete the first attribute
+>>> c1.name()
+'will tell'
+>>> c1.first = 'drew'
+>>> c1.first
'drew'
->>> del n1.number
+>>> del c1.number
Traceback (most recent call last):
...
TypeError: can't delete numeric/char attribute
->>> n1.number=2
->>> n1.number
+>>> c1.number=2
+>>> c1.number
2
->>> n1.first = 42
->>> n1.name()
-'42 tell'
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2 = noddy4.Noddy()
->>> n2.name()
+>>> c1.first = 42
+Traceback (most recent call last):
+...
+TypeError: The first attribute value must be a string
+>>> c1.name()
+'drew tell'
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2 = custom4.Custom()
+>>> c2.name()
' '
->>> n2.first
+>>> c2.first
''
->>> n2.last
+>>> c2.last
''
->>> del n2.first
->>> n2.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n2.first
-Traceback (most recent call last):
-...
-AttributeError: first
->>> n2.name()
-Traceback (most recent call last):
- File "", line 1, in ?
-AttributeError: first
->>> n2.number
+>>> c2.number
0
->>> n3 = noddy4.Noddy('jim', 'fulton', 'waaa')
+>>> n3 = custom4.Custom('jim', 'fulton', 'waaa')
Traceback (most recent call last):
- File "", line 1, in ?
-TypeError: an integer is required
+...
+TypeError: an integer is required (got type str)
Test cyclic gc(?)
@@ -183,15 +170,14 @@
>>> import gc
>>> gc.disable()
->>> x = []
->>> l = [x]
->>> n2.first = l
->>> n2.first
-[[]]
->>> l.append(n2)
->>> del l
->>> del n1
->>> del n2
+>>> class Subclass(custom4.Custom): pass
+...
+>>> s = Subclass()
+>>> s.cycle = [s]
+>>> s.cycle.append(s.cycle)
+>>> x = object()
+>>> s.x = x
+>>> del s
>>> sys.getrefcount(x)
3
>>> ignore = gc.collect()
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index bc080b00c59a82e..5a59b4d80388b9e 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -283,7 +283,9 @@ Windows, choose :menuselection:`Start --> Programs --> Python X.Y -->
Python (command line)`. Once the interpreter is started, you type Python code
at the prompt. For example, on my Linux system, I type the three Python
statements shown below, and get the output as shown, to find out my
-:file:`{prefix}` and :file:`{exec-prefix}`::
+:file:`{prefix}` and :file:`{exec-prefix}`:
+
+.. code-block:: pycon
Python 2.4 (#26, Aug 7 2004, 17:19:02)
Type "help", "copyright", "credits" or "license" for more information.
@@ -552,10 +554,10 @@ C headers ``--install-headers``
These override options can be relative, absolute,
or explicitly defined in terms of one of the installation base directories.
-(There are two installation base directories, and they are normally the same---
-they only differ when you use the Unix "prefix scheme" and supply different
-``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` will
-override values computed or given for ``--install-purelib`` and
+(There are two installation base directories, and they are normally the
+same---they only differ when you use the Unix "prefix scheme" and supply
+different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib``
+will override values computed or given for ``--install-purelib`` and
``--install-platlib``, and is recommended for schemes that don't make a
difference between Python and extension modules.)
@@ -582,10 +584,10 @@ in this case.)
If you maintain Python on Windows, you might want third-party modules to live in
a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}`
-itself. This is almost as easy as customizing the script installation directory
----you just have to remember that there are two types of modules to worry about,
-Python and extension modules, which can conveniently be both controlled by one
-option::
+itself. This is almost as easy as customizing the script installation
+directory---you just have to remember that there are two types of modules
+to worry about, Python and extension modules, which can conveniently be both
+controlled by one option::
python setup.py install --install-lib=Site
@@ -622,7 +624,9 @@ parsing your configuration file(s).
Obviously, specifying the entire installation scheme every time you install a
new module distribution would be very tedious. Thus, you can put these options
-into your Distutils config file (see section :ref:`inst-config-files`)::
+into your Distutils config file (see section :ref:`inst-config-files`):
+
+.. code-block:: ini
[install]
install-base=$HOME
@@ -631,7 +635,9 @@ into your Distutils config file (see section :ref:`inst-config-files`)::
install-scripts=python/scripts
install-data=python/data
-or, equivalently, ::
+or, equivalently,
+
+.. code-block:: ini
[install]
install-base=$HOME/python
@@ -718,7 +724,9 @@ A slightly less convenient way is to edit the :file:`site.py` file in Python's
standard library, and modify ``sys.path``. :file:`site.py` is automatically
imported when the Python interpreter is executed, unless the :option:`-S` switch
is supplied to suppress this behaviour. So you could simply edit
-:file:`site.py` and add two lines to it::
+:file:`site.py` and add two lines to it:
+
+.. code-block:: python
import sys
sys.path.append('/www/python/')
@@ -839,7 +847,9 @@ plus a ``global`` section for global options that affect every command. Each
section consists of one option per line, specified as ``option=value``.
For example, the following is a complete config file that just forces all
-commands to run quietly by default::
+commands to run quietly by default:
+
+.. code-block:: ini
[global]
verbose=0
@@ -853,7 +863,9 @@ distribution.
You could override the default "build base" directory and make the
:command:`build\*` commands always forcibly rebuild all files with the
-following::
+following:
+
+.. code-block:: ini
[build]
build-base=blib
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
index b22465df2910eb0..18f5b29396c9c26 100644
--- a/Doc/installing/index.rst
+++ b/Doc/installing/index.rst
@@ -44,7 +44,7 @@ Key terms
``venv``. It allows virtual environments to be used on versions of
Python prior to 3.4, which either don't provide ``venv`` at all, or
aren't able to automatically install ``pip`` into created environments.
-* The `Python Packaging Index `__ is a public
+* The `Python Packaging Index `__ is a public
repository of open source licensed packages made available for use by
other Python users.
* the `Python Packaging Authority
@@ -211,6 +211,17 @@ On such systems, it is often better to use a virtual environment or a
per-user installation when installing packages with ``pip``.
+Pip not installed
+-----------------
+
+It is possible that ``pip`` does not get installed by default. One potential fix is::
+
+ python -m ensurepip --default-pip
+
+There are also additional resources for `installing pip.
+`__
+
+
Installing binary extensions
----------------------------
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
index ace1bfaf8cb9d6d..1ab05a62ad458fb 100644
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -199,13 +199,6 @@ and off individually. They are described here in more detail.
because the :class:`memoryview` API is similar but not exactly the same as
that of :class:`buffer`.
-.. 2to3fixer:: callable
-
- Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding
- an import to :mod:`collections` if needed. Note ``callable(x)`` has returned
- in Python 3.2, so if you do not intend to support Python 3.1, you can disable
- this fixer.
-
.. 2to3fixer:: dict
Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to
@@ -295,7 +288,8 @@ and off individually. They are described here in more detail.
Fixes duplicate types in the second argument of :func:`isinstance`. For
example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x,
- (int))``.
+ int)`` and ``isinstance(x, (int, float, int))`` is converted to
+ ``isinstance(x, (int, float))``.
.. 2to3fixer:: itertools_imports
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index 0d2d818f5f90b00..06e2c02afc1e8b5 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -57,8 +57,12 @@ It defines the following constants and functions:
.. function:: interrupt_main()
- Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can
- use this function to interrupt the main thread.
+ Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main
+ thread. A thread can use this function to interrupt the main thread.
+
+ If :data:`signal.SIGINT` isn't handled by Python (it was set to
+ :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+ nothing.
.. function:: exit()
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 966003bd45ada1f..fc39a2e7582b3a8 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -18,13 +18,39 @@ see the PEP for why this was added to Python. (See also :pep:`3141` and the
:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.)
The :mod:`collections` module has some concrete classes that derive from
-ABCs; these can, of course, be further derived. In addition the
+ABCs; these can, of course, be further derived. In addition, the
:mod:`collections.abc` submodule has some ABCs that can be used to test whether
-a class or instance provides a particular interface, for example, is it
-hashable or a mapping.
+a class or instance provides a particular interface, for example, if it is
+hashable or if it is a mapping.
-This module provides the following classes:
+This module provides the metaclass :class:`ABCMeta` for defining ABCs and
+a helper class :class:`ABC` to alternatively define ABCs through inheritance:
+
+.. class:: ABC
+
+ A helper class that has :class:`ABCMeta` as its metaclass. With this class,
+ an abstract base class can be created by simply deriving from :class:`ABC`
+ avoiding sometimes confusing metaclass usage, for example::
+
+ from abc import ABC
+
+ class MyABC(ABC):
+ pass
+
+ Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore
+ inheriting from :class:`ABC` requires the usual precautions regarding
+ metaclass usage, as multiple inheritance may lead to metaclass conflicts.
+ One may also define an abstract base class by passing the metaclass
+ keyword and using :class:`ABCMeta` directly, for example::
+
+ from abc import ABCMeta
+
+ class MyABC(metaclass=ABCMeta):
+ pass
+
+ .. versionadded:: 3.4
+
.. class:: ABCMeta
@@ -46,15 +72,15 @@ This module provides the following classes:
Register *subclass* as a "virtual subclass" of this ABC. For
example::
- from abc import ABCMeta
+ from abc import ABC
- class MyABC(metaclass=ABCMeta):
- pass
+ class MyABC(ABC):
+ pass
- MyABC.register(tuple)
+ MyABC.register(tuple)
- assert issubclass(tuple, MyABC)
- assert isinstance((), MyABC)
+ assert issubclass(tuple, MyABC)
+ assert isinstance((), MyABC)
.. versionchanged:: 3.3
Returns the registered subclass, to allow usage as a class decorator.
@@ -95,7 +121,7 @@ This module provides the following classes:
def get_iterator(self):
return iter(self)
- class MyIterable(metaclass=ABCMeta):
+ class MyIterable(ABC):
@abstractmethod
def __iter__(self):
@@ -132,20 +158,9 @@ This module provides the following classes:
available as a method of ``Foo``, so it is provided separately.
-.. class:: ABC
-
- A helper class that has :class:`ABCMeta` as its metaclass. With this class,
- an abstract base class can be created by simply deriving from :class:`ABC`,
- avoiding sometimes confusing metaclass usage.
-
- Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore
- inheriting from :class:`ABC` requires the usual precautions regarding metaclass
- usage, as multiple inheritance may lead to metaclass conflicts.
-
- .. versionadded:: 3.4
-The :mod:`abc` module also provides the following decorators:
+The :mod:`abc` module also provides the following decorator:
.. decorator:: abstractmethod
@@ -168,7 +183,7 @@ The :mod:`abc` module also provides the following decorators:
descriptors, it should be applied as the innermost decorator, as shown in
the following usage examples::
- class C(metaclass=ABCMeta):
+ class C(ABC):
@abstractmethod
def my_abstract_method(self, ...):
...
@@ -202,7 +217,7 @@ The :mod:`abc` module also provides the following decorators:
the descriptor must identify itself as abstract using
:attr:`__isabstractmethod__`. In general, this attribute should be ``True``
if any of the methods used to compose the descriptor are abstract. For
- example, Python's built-in property does the equivalent of::
+ example, Python's built-in :class:`property` does the equivalent of::
class Descriptor:
...
@@ -221,8 +236,15 @@ The :mod:`abc` module also provides the following decorators:
multiple-inheritance.
+The :mod:`abc` module also supports the following legacy decorators:
+
.. decorator:: abstractclassmethod
+ .. versionadded:: 3.2
+ .. deprecated:: 3.3
+ It is now possible to use :class:`classmethod` with
+ :func:`abstractmethod`, making this decorator redundant.
+
A subclass of the built-in :func:`classmethod`, indicating an abstract
classmethod. Otherwise it is similar to :func:`abstractmethod`.
@@ -230,20 +252,20 @@ The :mod:`abc` module also provides the following decorators:
is now correctly identified as abstract when applied to an abstract
method::
- class C(metaclass=ABCMeta):
+ class C(ABC):
@classmethod
@abstractmethod
def my_abstract_classmethod(cls, ...):
...
+
+.. decorator:: abstractstaticmethod
+
.. versionadded:: 3.2
.. deprecated:: 3.3
- It is now possible to use :class:`classmethod` with
+ It is now possible to use :class:`staticmethod` with
:func:`abstractmethod`, making this decorator redundant.
-
-.. decorator:: abstractstaticmethod
-
A subclass of the built-in :func:`staticmethod`, indicating an abstract
staticmethod. Otherwise it is similar to :func:`abstractmethod`.
@@ -251,34 +273,28 @@ The :mod:`abc` module also provides the following decorators:
is now correctly identified as abstract when applied to an abstract
method::
- class C(metaclass=ABCMeta):
+ class C(ABC):
@staticmethod
@abstractmethod
def my_abstract_staticmethod(...):
...
- .. versionadded:: 3.2
- .. deprecated:: 3.3
- It is now possible to use :class:`staticmethod` with
- :func:`abstractmethod`, making this decorator redundant.
+.. decorator:: abstractproperty
-.. decorator:: abstractproperty(fget=None, fset=None, fdel=None, doc=None)
+ .. deprecated:: 3.3
+ It is now possible to use :class:`property`, :meth:`property.getter`,
+ :meth:`property.setter` and :meth:`property.deleter` with
+ :func:`abstractmethod`, making this decorator redundant.
A subclass of the built-in :func:`property`, indicating an abstract
property.
- Using this function requires that the class's metaclass is :class:`ABCMeta`
- or is derived from it. A class that has a metaclass derived from
- :class:`ABCMeta` cannot be instantiated unless all of its abstract methods
- and properties are overridden. The abstract properties can be called using
- any of the normal 'super' call mechanisms.
-
This special case is deprecated, as the :func:`property` decorator
is now correctly identified as abstract when applied to an abstract
method::
- class C(metaclass=ABCMeta):
+ class C(ABC):
@property
@abstractmethod
def my_abstract_property(self):
@@ -288,7 +304,7 @@ The :mod:`abc` module also provides the following decorators:
read-write abstract property by appropriately marking one or more of the
underlying methods as abstract::
- class C(metaclass=ABCMeta):
+ class C(ABC):
@property
def x(self):
...
@@ -307,12 +323,6 @@ The :mod:`abc` module also provides the following decorators:
...
- .. deprecated:: 3.3
- It is now possible to use :class:`property`, :meth:`property.getter`,
- :meth:`property.setter` and :meth:`property.deleter` with
- :func:`abstractmethod`, making this decorator redundant.
-
-
The :mod:`abc` module also provides the following functions:
.. function:: get_cache_token()
diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst
index 23a96207d083894..970a7aeb98a7139 100644
--- a/Doc/library/aifc.rst
+++ b/Doc/library/aifc.rst
@@ -18,12 +18,6 @@ AIFF is Audio Interchange File Format, a format for storing digital audio
samples in a file. AIFF-C is a newer version of the format that includes the
ability to compress the audio data.
-.. note::
-
- Some operations may only work under IRIX; these will raise :exc:`ImportError`
- when attempting to import the :mod:`cl` module, which is only available on
- IRIX.
-
Audio files have a number of parameters that describe the audio data. The
sampling rate or frame rate is the number of times per second the sound is
sampled. The number of channels indicate if the audio is mono, stereo, or
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index 45303048174dfab..5196202c47231b4 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -426,7 +426,9 @@ should not be line-wrapped::
-h, --help show this help message and exit
:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text,
-including argument descriptions.
+including argument descriptions. However, multiple new lines are replaced with
+one. If you wish to preserve multiple blank lines, add spaces between the
+newlines.
:class:`ArgumentDefaultsHelpFormatter` automatically adds information about
default values to each of the argument help messages::
@@ -710,7 +712,7 @@ be positional::
Namespace(bar='BAR', foo='FOO')
>>> parser.parse_args(['--foo', 'FOO'])
usage: PROG [-h] [-f FOO] bar
- PROG: error: too few arguments
+ PROG: error: the following arguments are required: bar
action
@@ -842,6 +844,8 @@ values are:
Note that ``nargs=1`` produces a list of one item. This is different from
the default, in which the item is produced by itself.
+.. index:: single: ? (question mark); in argparse module
+
* ``'?'``. One argument will be consumed from the command line if possible, and
produced as a single item. If no command-line argument is present, the value from
default_ will be produced. Note that for optional arguments, there is an
@@ -874,6 +878,8 @@ values are:
Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>,
outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>)
+.. index:: single: * (asterisk); in argparse module
+
* ``'*'``. All command-line arguments present are gathered into a list. Note that
it generally doesn't make much sense to have more than one positional argument
with ``nargs='*'``, but multiple optional arguments with ``nargs='*'`` is
@@ -886,6 +892,8 @@ values are:
>>> parser.parse_args('a b --foo x y --bar 1 2'.split())
Namespace(bar=['1', '2'], baz=['a', 'b'], foo=['x', 'y'])
+.. index:: single: + (plus); in argparse module
+
* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a
list. Additionally, an error message will be generated if there wasn't at
least one command-line argument present. For example::
@@ -896,7 +904,9 @@ values are:
Namespace(foo=['a', 'b'])
>>> parser.parse_args([])
usage: PROG [-h] foo [foo ...]
- PROG: error: too few arguments
+ PROG: error: the following arguments are required: foo
+
+.. _`argparse.REMAINDER`:
* ``argparse.REMAINDER``. All the remaining command-line arguments are gathered
into a list. This is commonly useful for command line utilities that dispatch
@@ -977,7 +987,7 @@ is used when no command-line argument was present::
Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the
-command-line argument was not present.::
+command-line argument was not present::
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('--foo', default=argparse.SUPPRESS)
@@ -1324,8 +1334,11 @@ The parse_args() method
created and how they are assigned. See the documentation for
:meth:`add_argument` for details.
- By default, the argument strings are taken from :data:`sys.argv`, and a new empty
- :class:`Namespace` object is created for the attributes.
+ * args_ - List of strings to parse. The default is taken from
+ :data:`sys.argv`.
+
+ * namespace_ - An object to take the attributes. The default is a new empty
+ :class:`Namespace` object.
Option value syntax
@@ -1467,6 +1480,7 @@ unambiguous (the prefix matches a unique option)::
An error is produced for arguments that could produce more than one options.
This feature can be disabled by setting :ref:`allow_abbrev` to ``False``.
+.. _args:
Beyond ``sys.argv``
^^^^^^^^^^^^^^^^^^^
@@ -1488,6 +1502,7 @@ interactive prompt::
>>> parser.parse_args(['1', '2', '3', '4', '--sum'])
Namespace(accumulate=, integers=[1, 2, 3, 4])
+.. _namespace:
The Namespace object
^^^^^^^^^^^^^^^^^^^^
@@ -2008,7 +2023,12 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`:
* Replace ``(options, args) = parser.parse_args()`` with ``args =
parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument`
calls for the positional arguments. Keep in mind that what was previously
- called ``options``, now in :mod:`argparse` context is called ``args``.
+ called ``options``, now in the :mod:`argparse` context is called ``args``.
+
+* Replace :meth:`optparse.OptionParser.disable_interspersed_args`
+ by setting ``nargs`` of a positional argument to `argparse.REMAINDER`_, or
+ use :meth:`~ArgumentParser.parse_known_args` to collect unparsed argument
+ strings in a separate list.
* Replace callback actions and the ``callback_*`` keyword arguments with
``type`` or ``action`` arguments.
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index 8d4ae2cc616fa3a..eb6675497f97343 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -41,6 +41,9 @@ Node classes
with alternatives (aka "sums"), the left-hand side class is abstract: only
instances of specific constructor nodes are ever created.
+ .. index:: single: ? (question mark); in AST grammar
+ .. index:: single: * (asterisk); in AST grammar
+
.. attribute:: _fields
Each concrete class has an attribute :attr:`_fields` which gives the names
@@ -113,6 +116,11 @@ and classes for traversing abstract syntax trees:
Parse the source into an AST node. Equivalent to ``compile(source,
filename, mode, ast.PyCF_ONLY_AST)``.
+ .. warning::
+ It is possible to crash the Python interpreter with a
+ sufficiently large/complex string due to stack depth limitations
+ in Python's AST compiler.
+
.. function:: literal_eval(node_or_string)
@@ -126,6 +134,11 @@ and classes for traversing abstract syntax trees:
capable of evaluating arbitrarily complex expressions, for example involving
operators or indexing.
+ .. warning::
+ It is possible to crash the Python interpreter with a
+ sufficiently large/complex string due to stack depth limitations
+ in Python's AST compiler.
+
.. versionchanged:: 3.2
Now allows bytes and set literals.
diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index cc4a14b2016226c..e2ad3eb0196c93c 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -52,7 +52,7 @@ Cancellation
------------
Cancellation of tasks is not common in classic programming. In asynchronous
-programming, not only it is something common, but you have to prepare your
+programming, not only is it something common, but you have to prepare your
code to handle it.
Futures and tasks can be cancelled explicitly with their :meth:`Future.cancel`
@@ -209,9 +209,9 @@ The fix is to call the :func:`ensure_future` function or the
Detect exceptions never consumed
--------------------------------
-Python usually calls :func:`sys.displayhook` on unhandled exceptions. If
+Python usually calls :func:`sys.excepthook` on unhandled exceptions. If
:meth:`Future.set_exception` is called, but the exception is never consumed,
-:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted
+:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted
` when the future is deleted by the garbage collector, with the
traceback where the exception was raised.
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
index fa6a29604ca2ba4..d053a60a9ea6c32 100644
--- a/Doc/library/asyncio-eventloop.rst
+++ b/Doc/library/asyncio-eventloop.rst
@@ -5,6 +5,8 @@
Base Event Loop
===============
+**Source code:** :source:`Lib/asyncio/events.py`
+
The event loop is the central execution device provided by :mod:`asyncio`.
It provides multiple facilities, including:
@@ -339,6 +341,23 @@ Creating connections
.. coroutinemethod:: AbstractEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0, reuse_address=None, reuse_port=None, allow_broadcast=None, sock=None)
+ .. note::
+ The parameter *reuse_address* is no longer supported, as using
+ :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for
+ UDP. Explicitly passing ``reuse_address=True`` will raise an exception.
+
+ When multiple processes with differing UIDs assign sockets to an
+ indentical UDP socket address with ``SO_REUSEADDR``, incoming packets can
+ become randomly distributed among the sockets.
+
+ For supported platforms, *reuse_port* can be used as a replacement for
+ similar functionality. With *reuse_port*,
+ :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically
+ prevents processes with differing UIDs from assigning sockets to the same
+ socket address.
+
+ Create a datagram connection.
+
Create datagram connection: socket family :py:data:`~socket.AF_INET` or
:py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
socket type :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a
@@ -363,11 +382,6 @@ Creating connections
resolution. If given, these should all be integers from the
corresponding :mod:`socket` module constants.
- * *reuse_address* tells the kernel to reuse a local socket in
- TIME_WAIT state, without waiting for its natural timeout to
- expire. If not specified will automatically be set to ``True`` on
- UNIX.
-
* *reuse_port* tells the kernel to allow this endpoint to be bound to the
same port as other existing endpoints are bound to, so long as they all
set this flag when being created. This option is not supported on Windows
@@ -387,6 +401,14 @@ Creating connections
See :ref:`UDP echo client protocol ` and
:ref:`UDP echo server protocol ` examples.
+ .. versionchanged:: 3.4.4
+ The *family*, *proto*, *flags*, *reuse_address*, *reuse_port,
+ *allow_broadcast*, and *sock* parameters were added.
+
+ .. versionchanged:: 3.6.10
+ The *reuse_address* parameter is no longer supporter due to security
+ concerns
+
.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None)
@@ -498,6 +520,9 @@ Creating listening connections
This method is a :ref:`coroutine `. When completed, the
coroutine returns a ``(transport, protocol)`` pair.
+ .. versionadded:: 3.5.3
+
+
Watch file descriptors
----------------------
diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst
index 1dc18fce79d021f..3aa9ffeff268c42 100644
--- a/Doc/library/asyncio-eventloops.rst
+++ b/Doc/library/asyncio-eventloops.rst
@@ -3,6 +3,8 @@
Event loops
===========
+**Source code:** :source:`Lib/asyncio/events.py`
+
Event loop functions
--------------------
@@ -146,10 +148,9 @@ process based on the calling context. A policy is an object implementing the
:class:`AbstractEventLoopPolicy` interface.
For most users of :mod:`asyncio`, policies never have to be dealt with
-explicitly, since the default global policy is sufficient.
+explicitly, since the default global policy is sufficient (see below).
-The default policy defines context as the current thread, and manages an event
-loop per thread that interacts with :mod:`asyncio`. The module-level functions
+The module-level functions
:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to
event loops managed by the default policy.
@@ -168,12 +169,15 @@ An event loop policy must implement the following interface:
Get the event loop for the current context.
Returns an event loop object implementing the :class:`AbstractEventLoop`
- interface.
+ interface. In case called from coroutine, it returns the currently
+ running event loop.
Raises an exception in case no event loop has been set for the current
context and the current policy does not specify to create one. It must
never return ``None``.
+ .. versionchanged:: 3.6
+
.. method:: set_event_loop(loop)
Set the event loop for the current context to *loop*.
@@ -187,6 +191,13 @@ An event loop policy must implement the following interface:
context, :meth:`set_event_loop` must be called explicitly.
+The default policy defines context as the current thread, and manages an event
+loop per thread that interacts with :mod:`asyncio`. If the current thread
+doesn't already have an event loop associated with it, the default policy's
+:meth:`~AbstractEventLoopPolicy.get_event_loop` method creates one when
+called from the main thread, but raises :exc:`RuntimeError` otherwise.
+
+
Access to the global loop policy
--------------------------------
@@ -198,3 +209,24 @@ Access to the global loop policy
Set the current event loop policy. If *policy* is ``None``, the default
policy is restored.
+
+
+Customizing the event loop policy
+---------------------------------
+
+To implement a new event loop policy, it is recommended you subclass the
+concrete default event loop policy :class:`DefaultEventLoopPolicy`
+and override the methods for which you want to change behavior, for example::
+
+ class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
+
+ def get_event_loop(self):
+ """Get the event loop.
+
+ This may be None or an instance of EventLoop.
+ """
+ loop = super().get_event_loop()
+ # Do something with loop ...
+ return loop
+
+ asyncio.set_event_loop_policy(MyEventLoopPolicy())
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
index 3fbf51058673d92..9605261c0a3f46c 100644
--- a/Doc/library/asyncio-protocol.rst
+++ b/Doc/library/asyncio-protocol.rst
@@ -1,8 +1,12 @@
.. currentmodule:: asyncio
-++++++++++++++++++++++++++++++++++++++++++++++
-Transports and protocols (callback based API)
-++++++++++++++++++++++++++++++++++++++++++++++
++++++++++++++++++++++++++++++++++++++++++++++
+Transports and protocols (callback based API)
++++++++++++++++++++++++++++++++++++++++++++++
+
+**Source code:** :source:`Lib/asyncio/transports.py`
+
+**Source code:** :source:`Lib/asyncio/protocols.py`
.. _asyncio-transport:
@@ -36,7 +40,7 @@ BaseTransport
Base class for transports.
- .. method:: close(self)
+ .. method:: close()
Close the transport. If the transport has a buffer for outgoing
data, buffered data will be flushed asynchronously. No more data
@@ -44,7 +48,7 @@ BaseTransport
protocol's :meth:`connection_lost` method will be called with
:const:`None` as its argument.
- .. method:: is_closing(self)
+ .. method:: is_closing()
Return ``True`` if the transport is closing or is closed.
@@ -120,11 +124,19 @@ ReadTransport
the protocol's :meth:`data_received` method until :meth:`resume_reading`
is called.
+ .. versionchanged:: 3.6.7
+ The method is idempotent, i.e. it can be called when the
+ transport is already paused or closed.
+
.. method:: resume_reading()
Resume the receiving end. The protocol's :meth:`data_received` method
will be called once again if some data is available for reading.
+ .. versionchanged:: 3.6.7
+ The method is idempotent, i.e. it can be called when the
+ transport is already reading.
+
WriteTransport
--------------
@@ -163,11 +175,17 @@ WriteTransport
Set the *high*- and *low*-water limits for write flow control.
- These two values control when call the protocol's
+ These two values (measured in number of
+ bytes) control when the protocol's
:meth:`pause_writing` and :meth:`resume_writing` methods are called.
If specified, the low-water limit must be less than or equal to the
high-water limit. Neither *high* nor *low* can be negative.
+ :meth:`pause_writing` is called when the buffer size becomes greater
+ than or equal to the *high* value. If writing has been paused,
+ :meth:`resume_writing` is called when the buffer size becomes less
+ than or equal to the *low* value.
+
The defaults are implementation-specific. If only the
high-water limit is given, the low-water limit defaults to an
implementation-specific value less than or equal to the
@@ -251,7 +269,7 @@ BaseSubprocessTransport
if it hasn't returned, similarly to the
:attr:`subprocess.Popen.returncode` attribute.
- .. method:: kill(self)
+ .. method:: kill()
Kill the subprocess, as in :meth:`subprocess.Popen.kill`.
@@ -384,7 +402,7 @@ The following callbacks are called on :class:`Protocol` instances:
.. method:: Protocol.eof_received()
- Calls when the other end signals it won't send any more data
+ Called when the other end signals it won't send any more data
(for example by calling :meth:`write_eof`, if the other end also uses
asyncio).
diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst
index f11c09ac29055e9..ea78755008244d8 100644
--- a/Doc/library/asyncio-queue.rst
+++ b/Doc/library/asyncio-queue.rst
@@ -3,6 +3,8 @@
Queues
======
+**Source code:** :source:`Lib/asyncio/queues.py`
+
Queues:
* :class:`Queue`
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
index 6177b4bb0f8b14f..a510e1e638a73d8 100644
--- a/Doc/library/asyncio-stream.rst
+++ b/Doc/library/asyncio-stream.rst
@@ -6,6 +6,8 @@
Streams (coroutine based API)
+++++++++++++++++++++++++++++
+**Source code:** :source:`Lib/asyncio/streams.py`
+
Stream functions
================
@@ -86,10 +88,12 @@ Stream functions
StreamReader
============
-.. class:: StreamReader(limit=None, loop=None)
+.. class:: StreamReader(limit=_DEFAULT_LIMIT, loop=None)
This class is :ref:`not thread safe `.
+ The *limit* argument's default value is set to _DEFAULT_LIMIT which is 2**16 (64 KiB)
+
.. method:: exception()
Get the exception.
diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
index dc93a74c6dee126..1c1d0be918da74c 100644
--- a/Doc/library/asyncio-subprocess.rst
+++ b/Doc/library/asyncio-subprocess.rst
@@ -5,6 +5,8 @@
Subprocess
==========
+**Source code:** :source:`Lib/asyncio/subprocess.py`
+
Windows event loop
------------------
@@ -80,7 +82,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module.
however, where :class:`~subprocess.Popen` takes a single argument which is
list of strings, :func:`subprocess_exec` takes multiple string arguments.
- The *protocol_factory* must instanciate a subclass of the
+ The *protocol_factory* must instantiate a subclass of the
:class:`asyncio.SubprocessProtocol` class.
Other parameters:
@@ -123,7 +125,7 @@ Run subprocesses asynchronously using the :mod:`subprocess` module.
using the platform's "shell" syntax. This is similar to the standard library
:class:`subprocess.Popen` class called with ``shell=True``.
- The *protocol_factory* must instanciate a subclass of the
+ The *protocol_factory* must instantiate a subclass of the
:class:`asyncio.SubprocessProtocol` class.
See :meth:`~AbstractEventLoop.subprocess_exec` for more details about
diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst
index 09093521524a89b..14e3defbf4118a6 100644
--- a/Doc/library/asyncio-sync.rst
+++ b/Doc/library/asyncio-sync.rst
@@ -4,6 +4,8 @@
Synchronization primitives
==========================
+**Source code:** :source:`Lib/asyncio/locks.py`
+
Locks:
* :class:`Lock`
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index 558d17c09697931..c0383331c1edf67 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -3,6 +3,10 @@
Tasks and coroutines
====================
+**Source code:** :source:`Lib/asyncio/tasks.py`
+
+**Source code:** :source:`Lib/asyncio/coroutines.py`
+
.. _coroutine:
Coroutines
@@ -212,7 +216,7 @@ Future
raise an exception when the future isn't done yet.
- Callbacks registered with :meth:`add_done_callback` are always called
- via the event loop's :meth:`~AbstractEventLoop.call_soon_threadsafe`.
+ via the event loop's :meth:`~AbstractEventLoop.call_soon`.
- This class is not compatible with the :func:`~concurrent.futures.wait` and
:func:`~concurrent.futures.as_completed` functions in the
@@ -540,6 +544,11 @@ Task functions
.. deprecated:: 3.4.4
+.. function:: wrap_future(future, \*, loop=None)
+
+ Wrap a :class:`concurrent.futures.Future` object in a :class:`Future`
+ object.
+
.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False)
Return a future aggregating results from the given coroutine objects or
@@ -559,6 +568,10 @@ Task functions
outer Future is *not* cancelled in this case. (This is to prevent the
cancellation of one child to cause other children to be cancelled.)
+ .. versionchanged:: 3.6.6
+ If the *gather* itself is cancelled, the cancellation is propagated
+ regardless of *return_exceptions*.
+
.. function:: iscoroutine(obj)
Return ``True`` if *obj* is a :ref:`coroutine object `,
@@ -621,7 +634,7 @@ Task functions
This function is a :ref:`coroutine `.
-.. function:: shield(arg, \*, loop=None)
+.. coroutinefunction:: shield(arg, \*, loop=None)
Wait for a future, shielding it from cancellation.
diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst
index 1d84d4587fd9c00..5c60b604c68301e 100644
--- a/Doc/library/atexit.rst
+++ b/Doc/library/atexit.rst
@@ -21,7 +21,7 @@ program is killed by a signal not handled by Python, when a Python fatal
internal error is detected, or when :func:`os._exit` is called.
-.. function:: register(func, *args, **kargs)
+.. function:: register(func, *args, **kwargs)
Register *func* as a function to be executed at termination. Any optional
arguments that are to be passed to *func* must be passed as arguments to
diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst
index 080d9d77ec984ea..ad9f5f58bee2aac 100644
--- a/Doc/library/base64.rst
+++ b/Doc/library/base64.rst
@@ -218,14 +218,6 @@ The modern interface provides:
.. versionadded:: 3.4
-.. note::
- Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or
- Ascii85 characters can encode 4 binary bytes), while the better-known
- Base64 has an expansion factor of 6 to 4. They are therefore more
- efficient when space expensive. They differ by details such as the
- character map used for encoding.
-
-
The legacy interface:
.. function:: decode(input, output)
@@ -237,14 +229,18 @@ The legacy interface:
.. function:: decodebytes(s)
- decodestring(s)
Decode the :term:`bytes-like object` *s*, which must contain one or more
lines of base64 encoded data, and return the decoded :class:`bytes`.
- ``decodestring`` is a deprecated alias.
.. versionadded:: 3.1
+.. function:: decodestring(s)
+
+ Deprecated alias of :func:`decodebytes`.
+
+ .. deprecated:: 3.1
+
.. function:: encode(input, output)
@@ -257,14 +253,19 @@ The legacy interface:
.. function:: encodebytes(s)
- encodestring(s)
Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary
data, and return :class:`bytes` containing the base64-encoded data, with newlines
(``b'\n'``) inserted after every 76 bytes of output, and ensuring that
there is a trailing newline, as per :rfc:`2045` (MIME).
- ``encodestring`` is a deprecated alias.
+ .. versionadded:: 3.1
+
+.. function:: encodestring(s)
+
+ Deprecated alias of :func:`encodebytes`.
+
+ .. deprecated:: 3.1
An example usage of the module:
diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst
index 0476f507717cda3..adb087ef8826bf0 100644
--- a/Doc/library/binascii.rst
+++ b/Doc/library/binascii.rst
@@ -77,7 +77,7 @@ The :mod:`binascii` module defines the following functions:
*quotetabs* is present and true, all tabs and spaces will be encoded. If the
optional argument *istext* is present and true, newlines are not encoded but
trailing whitespace will be encoded. If the optional argument *header* is
- present and true, spaces will be encoded as underscores per RFC1522. If the
+ present and true, spaces will be encoded as underscores per :rfc:`1522`. If the
optional argument *header* is present and false, newline characters will be
encoded as well; otherwise linefeed conversion might corrupt the binary data
stream.
diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst
index 359ab23b2f9787d..2966e0dbfbcfe87 100644
--- a/Doc/library/binhex.rst
+++ b/Doc/library/binhex.rst
@@ -55,5 +55,3 @@ the source for details.
If you code or decode textfiles on non-Macintosh platforms they will still use
the old Macintosh newline convention (carriage-return as end of line).
-As of this writing, :func:`hexbin` appears to not work in all cases.
-
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
index 41219eeaabae087..a3dad1b8b03188c 100644
--- a/Doc/library/cgi.rst
+++ b/Doc/library/cgi.rst
@@ -277,13 +277,12 @@ These are useful if you want more control, or if you want to employ some of the
algorithms implemented in this module in other circumstances.
-.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False)
+.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&")
Parse a query in the environment or from a file (the file defaults to
- ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are
+ ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are
passed to :func:`urllib.parse.parse_qs` unchanged.
-
.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False)
This function is deprecated in this module. Use :func:`urllib.parse.parse_qs`
@@ -308,6 +307,9 @@ algorithms implemented in this module in other circumstances.
Note that this does not parse nested multipart parts --- use
:class:`FieldStorage` for that.
+ .. versionchanged:: 3.6.13
+ Added the *separator* parameter.
+
.. function:: parse_header(string)
diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst
index f40cfdfd5921632..d57edb7eb1698d2 100644
--- a/Doc/library/cmd.rst
+++ b/Doc/library/cmd.rst
@@ -61,6 +61,10 @@ A :class:`Cmd` instance has the following methods:
An end-of-file on input is passed back as the string ``'EOF'``.
+ .. index::
+ single: ? (question mark); in a command interpreter
+ single: ! (exclamation); in a command interpreter
+
An interpreter instance will recognize a command name ``foo`` if and only if it
has a method :meth:`do_foo`. As a special case, a line beginning with the
character ``'?'`` is dispatched to the method :meth:`do_help`. As another
@@ -266,10 +270,10 @@ immediate playback::
'Draw circle with given radius an options extent and steps: CIRCLE 50'
circle(*parse(arg))
def do_position(self, arg):
- 'Print the current turle position: POSITION'
+ 'Print the current turtle position: POSITION'
print('Current position is %d %d\n' % position())
def do_heading(self, arg):
- 'Print the current turle heading in degrees: HEADING'
+ 'Print the current turtle heading in degrees: HEADING'
print('Current heading is %d\n' % (heading(),))
def do_color(self, arg):
'Set the color: COLOR BLUE'
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 0ecd89ba75b617b..992135154fd6c05 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -312,6 +312,14 @@ defined and implemented by all standard Python codecs:
The following error handlers are only applicable to
:term:`text encodings `:
+.. index::
+ single: ? (question mark); replacement character
+ single: \ (backslash); escape sequence
+ single: \x; escape sequence
+ single: \u; escape sequence
+ single: \U; escape sequence
+ single: \N; escape sequence
+
+-------------------------+-----------------------------------------------+
| Value | Meaning |
+=========================+===============================================+
@@ -554,19 +562,19 @@ define in order to be compatible with the Python codec registry.
if necessary, to reset the encoder and to get the output.
-.. method:: IncrementalEncoder.getstate()
+ .. method:: getstate()
- Return the current state of the encoder which must be an integer. The
- implementation should make sure that ``0`` is the most common state. (States
- that are more complicated than integers can be converted into an integer by
- marshaling/pickling the state and encoding the bytes of the resulting string
- into an integer).
+ Return the current state of the encoder which must be an integer. The
+ implementation should make sure that ``0`` is the most common
+ state. (States that are more complicated than integers can be converted
+ into an integer by marshaling/pickling the state and encoding the bytes
+ of the resulting string into an integer).
-.. method:: IncrementalEncoder.setstate(state)
+ .. method:: setstate(state)
- Set the state of the encoder to *state*. *state* must be an encoder state
- returned by :meth:`getstate`.
+ Set the state of the encoder to *state*. *state* must be an encoder state
+ returned by :meth:`getstate`.
.. _incremental-decoder-objects:
@@ -802,7 +810,7 @@ The design is such that one can use the factory functions returned by the
:func:`lookup` function to construct the instance.
-.. class:: StreamReaderWriter(stream, Reader, Writer, errors)
+.. class:: StreamReaderWriter(stream, Reader, Writer, errors='strict')
Creates a :class:`StreamReaderWriter` instance. *stream* must be a file-like
object. *Reader* and *Writer* must be factory functions or classes providing the
@@ -826,7 +834,7 @@ The design is such that one can use the factory functions returned by the
:func:`lookup` function to construct the instance.
-.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors)
+.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict')
Creates a :class:`StreamRecoder` instance which implements a two-way conversion:
*encode* and *decode* work on the frontend — the data visible to
@@ -977,10 +985,14 @@ e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec.
Some common encodings can bypass the codecs lookup machinery to
improve performance. These optimization opportunities are only
- recognized by CPython for a limited set of aliases: utf-8, utf8,
- latin-1, latin1, iso-8859-1, mbcs (Windows only), ascii, utf-16,
- and utf-32. Using alternative spellings for these encodings may
- result in slower execution.
+ recognized by CPython for a limited set of (case insensitive)
+ aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs
+ (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and
+ the same using underscores instead of dashes. Using alternative
+ aliases for these encodings may result in slower execution.
+
+ .. versionchanged:: 3.6
+ Optimization opportunity recognized for us-ascii.
Many of the character sets support the same languages. They vary in individual
characters (e.g. whether the EURO SIGN is supported or not), and in the
@@ -1109,10 +1121,10 @@ particular, the following variants typically exist:
| | ks_c-5601, ks_c-5601-1987, | |
| | ksx1001, ks_x-1001 | |
+-----------------+--------------------------------+--------------------------------+
-| gb2312 | chinese, csiso58gb231280, euc- | Simplified Chinese |
-| | cn, euccn, eucgb2312-cn, | |
-| | gb2312-1980, gb2312-80, iso- | |
-| | ir-58 | |
+| gb2312 | chinese, csiso58gb231280, | Simplified Chinese |
+| | euc-cn, euccn, eucgb2312-cn, | |
+| | gb2312-1980, gb2312-80, | |
+| | iso-ir-58 | |
+-----------------+--------------------------------+--------------------------------+
| gbk | 936, cp936, ms936 | Unified Chinese |
+-----------------+--------------------------------+--------------------------------+
@@ -1421,7 +1433,7 @@ to the user.
Python supports this conversion in several ways: the ``idna`` codec performs
conversion between Unicode and ACE, separating an input string into labels
-based on the separator characters defined in `section 3.1`_ (1) of :rfc:`3490`
+based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>`
and converting each label to ACE as required, and conversely separating an input
byte string into labels based on the ``.`` separator and converting any ACE
labels found into unicode. Furthermore, the :mod:`socket` module
@@ -1432,8 +1444,6 @@ parameters, such as :mod:`http.client` and :mod:`ftplib`, accept Unicode host
names (:mod:`http.client` then also transparently sends an IDNA hostname in the
:mailheader:`Host` field if it sends that field at all).
-.. _section 3.1: https://tools.ietf.org/html/rfc3490#section-3.1
-
When receiving host names from the wire (such as in reverse name lookup), no
automatic conversion to Unicode is performed: Applications wishing to present
such host names to the user should decode them to Unicode.
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
index 58b03b9bd768042..60154532094423b 100644
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -107,7 +107,12 @@ ABC Inherits from Abstract Methods Mixin
.. class:: Iterable
ABC for classes that provide the :meth:`__iter__` method.
- See also the definition of :term:`iterable`.
+
+ Checking ``isinstance(obj, Iterable)`` detects classes that are registered
+ as :class:`Iterable` or that have an :meth:`__iter__` method, but it does
+ not detect classes that iterate with the :meth:`__getitem__` method.
+ The only reliable way to determine whether an object is :term:`iterable`
+ is to call ``iter(obj)``.
.. class:: Collection
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 8e2eb4d9b0382f4..25687348d16deef 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -198,6 +198,7 @@ updates keys found deeper in the chain::
>>> d['lion'] = 'orange' # update an existing key two levels down
>>> d['snake'] = 'red' # new keys get added to the topmost dict
>>> del d['elephant'] # remove an existing key one level down
+ >>> d # display result
DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'})
@@ -509,11 +510,14 @@ or subtracting from an empty counter.
.. versionadded:: 3.2
- .. method:: rotate(n)
+ .. method:: rotate(n=1)
- Rotate the deque *n* steps to the right. If *n* is negative, rotate to
- the left. Rotating one step to the right is equivalent to:
- ``d.appendleft(d.pop())``.
+ Rotate the deque *n* steps to the right. If *n* is negative, rotate
+ to the left.
+
+ When the deque is not empty, rotating one step to the right is equivalent
+ to ``d.appendleft(d.pop())``, and rotating one step to the left is
+ equivalent to ``d.append(d.popleft())``.
Deque objects also provide one read-only attribute:
@@ -771,9 +775,9 @@ they add the ability to access fields by name instead of position index.
helpful docstring (with typename and field_names) and a helpful :meth:`__repr__`
method which lists the tuple contents in a ``name=value`` format.
- The *field_names* are a single string with each fieldname separated by whitespace
- and/or commas, for example ``'x y'`` or ``'x, y'``. Alternatively, *field_names*
- can be a sequence of strings such as ``['x', 'y']``.
+ The *field_names* are a sequence of strings such as ``['x', 'y']``.
+ Alternatively, *field_names* can be a single string with each fieldname
+ separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``.
Any valid Python identifier may be used for a fieldname except for names
starting with an underscore. Valid identifiers consist of letters, digits,
@@ -866,7 +870,7 @@ field names, the method and attribute names start with an underscore.
.. versionchanged:: 3.1
Returns an :class:`OrderedDict` instead of a regular :class:`dict`.
-.. method:: somenamedtuple._replace(kwargs)
+.. method:: somenamedtuple._replace(**kwargs)
Return a new instance of the named tuple replacing specified fields with new
values::
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index d85576b8bedd8ee..319f757aa9063d9 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -40,21 +40,29 @@ Executor Objects
.. method:: map(func, *iterables, timeout=None, chunksize=1)
- Equivalent to :func:`map(func, *iterables)