diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 668bb5aceb..9c2d9d9e4a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,5 +4,5 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format - -garbage_collector.rst @pablogsal +# PSRT member list owned by PSRT admins. +security/psrt*.csv @warsaw @ewdurbin @ned-deily @sethmlarson diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index b160c6ea11..f6d55b1247 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,7 +1,7 @@ name: "Bug report" description: Create a report to help us improve the Python devguide title: "Bug: " -labels: ["bug"] +labels: ["type-bug"] assignees: [] body: diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index a4413c137a..7a7e9ec805 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,7 +1,7 @@ name: "Feature request" description: Suggest an idea for the Python devguide title: "Feature: <title>" -labels: ["enhancement"] +labels: ["type-feature"] assignees: [] body: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 23f03616cf..1c0343afa3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,6 +10,11 @@ updates: actions: patterns: - "*" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: pip directory: "/" @@ -21,3 +26,5 @@ updates: pip: patterns: - "*" + cooldown: + default-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b85a45c1bb..e60db2ddca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,8 @@ on: [pull_request, push, workflow_dispatch] env: FORCE_COLOR: 1 +permissions: {} + jobs: test: name: Check build, markup, and links @@ -12,12 +14,14 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - name: Install uv - uses: hynek/setup-cached-uv@v2 + uses: hynek/setup-cached-uv@4300ec2180bc77d705e626a34e381b81a4772c51 # v2.5.0 - name: Build docs run: make html - name: Link check diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4622f995aa..a3972b812a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,14 +2,22 @@ name: Lint on: [push, pull_request, workflow_dispatch] +permissions: {} + +env: + FORCE_COLOR: 1 + RUFF_OUTPUT_FORMAT: github + jobs: lint: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5beee4a37..07fdda903c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.7 + rev: v0.15.14 hooks: - id: ruff name: Run Ruff (lint) @@ -9,7 +9,7 @@ repos: name: Run Ruff (format) - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -18,8 +18,13 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.25.2 + hooks: + - id: zizmor + - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.9.1 + rev: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/.readthedocs.yml b/.readthedocs.yml index 26e5be9672..3e13033540 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,9 +13,14 @@ build: tools: python: "3" - commands: - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make dirhtml BUILDDIR=_readthedocs - - mv _readthedocs/dirhtml _readthedocs/html + jobs: + post_checkout: + - git fetch --unshallow || true + build: + html: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + - make dirhtml BUILDDIR=_readthedocs + - mkdir -p $READTHEDOCS_OUTPUT + - mv _readthedocs/dirhtml $READTHEDOCS_OUTPUT/html diff --git a/_static/devguide_overrides.css b/_static/devguide_overrides.css index 625a9dda2d..abe61bcdcd 100644 --- a/_static/devguide_overrides.css +++ b/_static/devguide_overrides.css @@ -27,3 +27,9 @@ table.docutils td { vertical-align: top; } + +/* For the video in index.rst */ +[role="main"] .compact-video { + max-width: 500px; + margin-inline: auto; +} diff --git a/_static/psrt-coordinator-report.html b/_static/psrt-coordinator-report.html new file mode 100644 index 0000000000..d1e3c9d695 --- /dev/null +++ b/_static/psrt-coordinator-report.html @@ -0,0 +1,31 @@ +<!-- Centering a div --> +<div style="width: 100%; text-align: center;"> +<div style="display: inline-block; text-align: left;"> +<code><pre> + ┌────────────────────────┐ + │ Received a report... │ + └──┬──────────────────┬──┘ + │ │ +┌────────────▼──────────────┐ ┌─▼──────────┐ +│ ..via security@python.org │ │ ..via GHSA │ +└────────────┬──────────────┘ └─┬──────────┘ + │ │ +┌────────────▼──────────────┐ │ +│ PSRT admin reviews email │ │ +└───┬───────────────────┬───┘ │ + │ │ │ +┌───▼────┐ ┌────▼───┐ │ +│ Reject │ │ Accept │ │ +└────────┘ └────┬───┘ │ + │ │ +┌───────────────────────▼───┐ │ +│ PSRT asks reporter to │ │ +│ open via GHSA │ │ +└────────────┬──────────────┘ │ + │ │ + ┌──▼──────────────────▼──┐ + │ GHSA in state 'Triage' │ + └────────────────────────┘ +</pre></code> +</div> +</div> diff --git a/_static/psrt-coordinator-triage.html b/_static/psrt-coordinator-triage.html new file mode 100644 index 0000000000..af5936a6cb --- /dev/null +++ b/_static/psrt-coordinator-triage.html @@ -0,0 +1,40 @@ +<!-- Centering a div --> +<div style="width: 100%; text-align: center;"> +<div style="display: inline-block; text-align: left;"> +<code><pre> + ┌────────────────────────┐ + │ GHSA in state 'Triage' │ + └─┬────────────────────┬─┘ + │ │ +┌──────────────────▼─────────┐ ┌────────▼───────────────────┐ +│ │ │ After 3 days idle, │ +│ PSRT member volunteers │ │ Coordinator is assigned │ +│ as Coordinator │ │ automatically by bot │ +└──────────────────┬─────────┘ └────────┬───────────────────┘ + │ │ + ┌─▼────────────────────▼─┐ + │ GHSA update 'Credit' │ + │ field to Coordinator │ + └────────────┬───────────┘ + │ + ┌────────────▼───────────┐ + │ Coordinator determines │ + ┌───┤ whether report... ┼───┐ + │ └────────────┬───────────┘ │ + │ │ │ +┌────────────▼─┐ ┌────────────▼───────────┐ ┌─▼─────────────┐ +│..is a invalid│ │ ..is a bug, but not │ │..is a security│ +│ or non-issue│ │ security relevant │ │ issue │ +└────────────┬─┘ └────────────┬───────────┘ └─┬─────────────┘ + │ │ │ + │ ┌────────────▼───────────┐ │ + │ │ Coordinator opens a │ │ + │ │ public GitHub issue │ │ + │ └─┬──────────────────────┘ │ + │ │ │ + ┌──────────▼─────▼───────┐ ┌────────────▼───────────┐ + │ Close the GHSA ticket │ │ GHSA in state 'Draft' │ + └────────────────────────┘ └────────────────────────┘ +</pre></code> +</div> +</div> diff --git a/_templates/page.html b/_templates/page.html new file mode 100644 index 0000000000..d1c9a515a4 --- /dev/null +++ b/_templates/page.html @@ -0,0 +1,8 @@ +{% extends "!page.html" %} + +{% block extrahead %} +{{ super() }} +{% if not is_deployment_preview %} +<script defer data-domain="devguide.python.org" src="https://analytics.python.org/js/script.outbound-links.js"></script> +{% endif %} +{% endblock %} diff --git a/_tools/release_cycle_template.svg.jinja b/_tools/release_cycle_template.svg.jinja index 9f23814a9e..ffc54919e7 100644 --- a/_tools/release_cycle_template.svg.jinja +++ b/_tools/release_cycle_template.svg.jinja @@ -26,6 +26,9 @@ width: 100%; --blob-border-width: 1.6px; + + --status-bg-color: #CCC; + --status-border-color: #888; } .background { fill: var(--svg-color-background-primary); @@ -88,6 +91,9 @@ --status-bg-color: #2222EE; --status-border-color: #008888; } + .release-cycle-status-planned { + /* Use grey defaults */ + } .release-cycle-blob { fill: var(--status-bg-color); stroke: transparent; diff --git a/conf.py b/conf.py index c38e33dd85..020e6bf6af 100644 --- a/conf.py +++ b/conf.py @@ -1,17 +1,23 @@ import json +import os from urllib.request import urlopen extensions = [ + 'linklint.ext', 'notfound.extension', 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx_copybutton', 'sphinx_inline_tabs', + 'sphinx_last_updated_by_git', + 'sphinxcontrib.youtube', 'sphinxext.opengraph', 'sphinxext.rediraffe', ] +html_last_updated_fmt = '%b %d, %Y' + # The master toctree document. master_doc = 'index' @@ -37,6 +43,7 @@ "source_repository": "https://github.com/python/devguide", "source_branch": "main", } +templates_path = ['_templates'] html_static_path = ['_static'] html_css_files = [ 'devguide_overrides.css', @@ -50,6 +57,14 @@ # Set to '' to prevent appending "documentation" to the site title html_title = "" +# Deployment preview information +# (See .readthedocs.yaml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) +is_deployment_preview = os.getenv("READTHEDOCS_VERSION_TYPE") == "external" + +html_context = { + "is_deployment_preview": is_deployment_preview, +} + linkcheck_allowed_redirects = { # Edit page r"https://docs.google.com/document/d/.*/": r"https://docs.google.com/document/d/.*/edit", # noqa: E501 @@ -136,6 +151,9 @@ "langchanges.rst": "developer-workflow/lang-changes.rst", "porting.rst": "developer-workflow/porting.rst", "stdlibchanges.rst": "developer-workflow/stdlib.rst", + # Security + "developer-workflow/psrt.rst": "security/psrt.rst", + "developer-workflow/sbom.rst": "security/sbom.rst", # Documentation "docquality.rst": "documentation/help-documenting.rst", "documenting.rst": "documentation/start-documenting.rst", @@ -149,6 +167,7 @@ "gitbootcamp.rst": "getting-started/git-boot-camp.rst", "pullrequest.rst": "getting-started/pull-request-lifecycle.rst", "setup.rst": "getting-started/setup-building.rst", + "getting-started/generative-ai.rst": "getting-started/ai-tools.rst", # CPython Internals "compiler.rst": "internals.rst", "exploring.rst": "internals.rst", @@ -169,6 +188,56 @@ "tracker.rst": "triage/issue-tracker.rst", "gh-labels.rst": "triage/labels.rst", "triaging.rst": "triage/triaging.rst", + # Contributing guide draft pages + "contrib/code/git.rst": "getting-started/git-boot-camp.rst", + "contrib/code/pull-request-lifecycle.rst": "getting-started/pull-request-lifecycle.rst", + "contrib/code/setup.rst": "getting-started/setup-building.rst", + "contrib/code/testing.rst": "testing/index.rst", + "contrib/code/developer-workflow.rst": "developer-workflow/index.rst", + "contrib/code/index.rst": "index.rst", + "contrib/code/development-tools.rst": "development-tools/index.rst", + "contrib/doc/devguide.rst": "documentation/devguide.rst", + "contrib/doc/help-documenting.rst": "documentation/help-documenting.rst", + "contrib/doc/markup.rst": "documentation/markup.rst", + "contrib/doc/pull-request-lifecycle.rst": "getting-started/pull-request-lifecycle.rst", + "contrib/doc/start-documenting.rst": "documentation/start-documenting.rst", + "contrib/doc/style-guide.rst": "documentation/style-guide.rst", + "contrib/doc/translating.rst": "documentation/translations/index.rst", + "contrib/doc/index.rst": "documentation/index.rst", + "contrib/intro/index.rst": "index.rst", + "contrib/project/channels.rst": "developer-workflow/communication-channels.rst", + "contrib/project/conduct.rst": "index.rst", + "contrib/project/github.rst": "index.rst", + "contrib/project/governance.rst": "index.rst", + "contrib/project/roles.rst": "index.rst", + "contrib/project/generative-ai": "getting-started/ai-tools.rst", + "contrib/project/outreach.rst": "index.rst", + "contrib/project/directory-structure.rst": "getting-started/setup-building.rst", + "contrib/project/index.rst": "index.rst", + "contrib/security.rst": "index.rst", + "contrib/triage/issue-tracker.rst": "triage/issue-tracker.rst", + "contrib/triage/labels.rst": "triage/labels.rst", + "contrib/triage/reviewing.rst": "triage/triaging.rst", + "contrib/triage/triage-team.rst": "triage/triage-team.rst", + "contrib/triage/triaging.rst": "triage/triaging.rst", + "contrib/triage/index.rst": "triage/index.rst", + "contrib/user-success.rst": "index.rst", + "contrib/core-team/committing.rst": "core-team/committing.rst", + "contrib/core-team/experts.rst": "core-team/experts.rst", + "contrib/core-team/index.rst": "core-team/index.rst", + "contrib/core-team/join-team.rst": "core-team/join-team.rst", + "contrib/core-team/motivations.rst": "core-team/motivations.rst", + "contrib/core-team/responsibilities.rst": "core-team/responsibilities.rst", + "contrib/core-team/team-log.rst": "core-team/team-log.rst", + "contrib/workflows/codespaces.rst": "getting-started/setup-building.rst", + "contrib/workflows/compile.rst": "getting-started/setup-building.rst", + "contrib/workflows/get-source.rst": "getting-started/setup-building.rst", + "contrib/workflows/index.rst": "getting-started/setup-building.rst", + "contrib/workflows/install-dependencies.rst": "getting-started/setup-building.rst", + "contrib/workflows/install-git.rst": "getting-started/setup-building.rst", + "contrib/workflows/regenerate.rst": "getting-started/setup-building.rst", + "contrib/workflows/troubleshooting.rst": "getting-started/setup-building.rst", + "contrib/index.rst": "index.rst", } intersphinx_mapping = { @@ -192,21 +261,6 @@ # prolog and epilogs rst_prolog = f""" -.. |draft| replace:: - This is part of a **Draft** of the Python Contributor's Guide. - Text in square brackets are notes about content to fill in. - Currently, the devguide and this new Contributor's Guide co-exist in the - repo. We are using Sphinx include directives to demonstrate the re-organization. - The final Contributor's Guide will replace the devguide with content in only one - place. - We welcome help with this! - -.. |purpose| replace:: - The :ref:`contrib-plan` page has more details about the current state of this draft - and **how you can help**. See more info about the Contributor Guide in the - discussion forum: `Refactoring the DevGuide`_. - -.. _Refactoring the DevGuide: https://discuss.python.org/t/refactoring-the-devguide-into-a-contribution-guide/63409 .. |main_version| replace:: {_main_version} diff --git a/contrib/code/developer-workflow.rst b/contrib/code/developer-workflow.rst deleted file mode 100644 index 416ca2c022..0000000000 --- a/contrib/code/developer-workflow.rst +++ /dev/null @@ -1,25 +0,0 @@ -==================== -Development workflow -==================== - -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`dev-workflow` page from the devguide] - -.. toctree:: - :maxdepth: 5 - - ../../developer-workflow/communication-channels - ../../developer-workflow/development-cycle - ../../developer-workflow/stdlib - ../../developer-workflow/extension-modules - ../../developer-workflow/c-api - ../../developer-workflow/lang-changes - ../../developer-workflow/grammar - ../../developer-workflow/porting - ../../developer-workflow/sbom - ../../developer-workflow/psrt diff --git a/contrib/code/development-tools.rst b/contrib/code/development-tools.rst deleted file mode 100644 index 769bba2b06..0000000000 --- a/contrib/code/development-tools.rst +++ /dev/null @@ -1,19 +0,0 @@ -================= -Development tools -================= - -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`development-tools` page from the devguide.] - -.. toctree:: - :maxdepth: 5 - - ../../development-tools/clinic/index - ../../development-tools/gdb - ../../development-tools/clang - ../../development-tools/warnings diff --git a/contrib/code/git.rst b/contrib/code/git.rst deleted file mode 100644 index 7c7aaa57b1..0000000000 --- a/contrib/code/git.rst +++ /dev/null @@ -1,11 +0,0 @@ -======== -Git tips -======== - -.. important:: - - |draft| - - |purpose| - -[More git help for advanced things needed by code contributors.] diff --git a/contrib/code/index.rst b/contrib/code/index.rst deleted file mode 100644 index 7680950663..0000000000 --- a/contrib/code/index.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _c_code: - -================== -Code contributions -================== - -.. important:: - - |draft| - - |purpose| - -[The main page for code contributors.] - -[We'll include code-focused content from the :ref:`main devguide page <devguide-main>`: Quick -reference, Quick links, Proposing changes, and so on.] - -[The existing :ref:`internals` section of the devguide will be fully -migrated into the Python repo.] - - -.. toctree:: - :maxdepth: 5 - - setup - git - pull-request-lifecycle - developer-workflow - testing - development-tools diff --git a/contrib/code/pull-request-lifecycle.rst b/contrib/code/pull-request-lifecycle.rst deleted file mode 100644 index 30c0fd5903..0000000000 --- a/contrib/code/pull-request-lifecycle.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _code-pull-request-lifecycle: - -====================== -Pull request lifecycle -====================== - -.. important:: - - |draft| - - |purpose| - - -[Details of pull requests for code contributions. The existing -:ref:`pull-request-lifecycle` page is long and includes many details. -Some only apply to code contributions, but many are common to all -contributions. Should we keep a common page, with extra steps here, or -should this page have all of the details even if they are duplicated -elsewhere?] - -[See :ref:`docs-pull-request-lifecycle` for the documentation half of this conundrum.] diff --git a/contrib/code/setup.rst b/contrib/code/setup.rst deleted file mode 100644 index 2d14bb0d91..0000000000 --- a/contrib/code/setup.rst +++ /dev/null @@ -1,12 +0,0 @@ -================== -Setup and building -================== - -.. important:: - - |draft| - - |purpose| - -[More setup and build instructions specifically for code contributors, building -on the basics from the :ref:`Getting Started <getting-started>` section.] diff --git a/contrib/code/testing.rst b/contrib/code/testing.rst deleted file mode 100644 index 575d1477a4..0000000000 --- a/contrib/code/testing.rst +++ /dev/null @@ -1,20 +0,0 @@ -===================== -Testing and buildbots -===================== - -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`testing` page from the devguide.] - -.. toctree:: - :maxdepth: 5 - - ../../testing/run-write-tests - ../../testing/silence-warnings - ../../testing/coverage - ../../testing/buildbots - ../../testing/new-buildbot-worker diff --git a/contrib/contrib-plan.rst b/contrib/contrib-plan.rst deleted file mode 100644 index 65e386e2b9..0000000000 --- a/contrib/contrib-plan.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _contrib-plan: - -================================== -[Plan for the Contributor's Guide] -================================== - -.. important:: - - |draft| - - |purpose| - -We are in the process of updating and refactoring the devguide to be a -Contributor's Guide. It will highlight the different kinds of contribution -possible, and how to succeed at each kind. - -Currently, the Contibutor's Guide is a draft in this new last section of the -devguide. We welcome feedback, but please understand that some of the current -content is moving or skeletal. - -Repo structure -============== - -While the reorganization is happening, we are keeping the old devguide as it -is. The new Contributor's Guide is represented in this last section, but will -eventually be the only content in the guide. To avoid copying content, we're -using Sphinx include directives to display existing devguide content in its new -Contributor's Guide location. That is not how the eventual Guide will be -built. Once we are ready to make the Contributor's Guide real, we will -rearrange content into its new location. - -How to help -=========== - -To help, you can: - -- `Write an issue`_ detailing a change you'd like to see here. -- `Make a pull request`_ in this repo to add content. -- Join us in the `Python Docs Discord`_ to collaborate with other docs-minded - community members. -- Get in touch with the `Docs Editorial Board`_ to discuss larger documentation - concerns. - -.. _Write an issue: https://github.com/python/devguide/issues -.. _Make a pull request: https://github.com/python/devguide/pulls -.. _Python Docs Discord: https://discord.gg/qcfPbnM2zH -.. _Docs Editorial Board: https://python.github.io/editorial-board/ diff --git a/contrib/core-team/committing.rst b/contrib/core-team/committing.rst deleted file mode 100644 index 5b639cd5a0..0000000000 --- a/contrib/core-team/committing.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core developers :ref:`committing` page from the devguide. We'll -adjust "core developer" to "core team" where appropriate.] - -.. include:: ../../core-team/committing.rst diff --git a/contrib/core-team/experts.rst b/contrib/core-team/experts.rst deleted file mode 100644 index aa16f10bd9..0000000000 --- a/contrib/core-team/experts.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core team :ref:`experts` page from the devguide.] - -.. include:: ../../core-team/experts.rst diff --git a/contrib/core-team/index.rst b/contrib/core-team/index.rst deleted file mode 100644 index 2ca21344b0..0000000000 --- a/contrib/core-team/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -.. _c_core-team: - -========= -Core team -========= - -[This is mostly re-organized from the :ref:`core-team` section of the devguide] - -.. toctree:: - :maxdepth: 5 - - responsibilities - committing - experts - team-log - motivations - join-team diff --git a/contrib/core-team/join-team.rst b/contrib/core-team/join-team.rst deleted file mode 100644 index 932216f7c6..0000000000 --- a/contrib/core-team/join-team.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core team :ref:`join-core-team` page from the devguide.] - -.. include:: ../../core-team/join-team.rst diff --git a/contrib/core-team/motivations.rst b/contrib/core-team/motivations.rst deleted file mode 100644 index 38ba310236..0000000000 --- a/contrib/core-team/motivations.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core team :ref:`motivations` page from the devguide.] - -.. include:: ../../core-team/motivations.rst diff --git a/contrib/core-team/responsibilities.rst b/contrib/core-team/responsibilities.rst deleted file mode 100644 index d6902bd780..0000000000 --- a/contrib/core-team/responsibilities.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core team :ref:`responsibilities` page from the devguide.] - -.. include:: ../../core-team/responsibilities.rst diff --git a/contrib/core-team/team-log.rst b/contrib/core-team/team-log.rst deleted file mode 100644 index ecfe856a41..0000000000 --- a/contrib/core-team/team-log.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing core team :ref:`team-log` page from the devguide.] - -.. include:: ../../core-team/team-log.rst diff --git a/contrib/doc/devguide.rst b/contrib/doc/devguide.rst deleted file mode 100644 index 2c83e52003..0000000000 --- a/contrib/doc/devguide.rst +++ /dev/null @@ -1,12 +0,0 @@ -================================== -Helping with the Developer's Guide -================================== - -.. important:: - - |draft| - - |purpose| - - -[This is the existing :ref:`devguide` page from the devguide.] diff --git a/contrib/doc/help-documenting.rst b/contrib/doc/help-documenting.rst deleted file mode 100644 index befb4b2461..0000000000 --- a/contrib/doc/help-documenting.rst +++ /dev/null @@ -1,12 +0,0 @@ -========================== -Helping with documentation -========================== - -.. important:: - - |draft| - - |purpose| - - -[This is the existing :ref:`help-documenting` page from the devguide.] diff --git a/contrib/doc/index.rst b/contrib/doc/index.rst deleted file mode 100644 index dc8ec93074..0000000000 --- a/contrib/doc/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _c_docs: - -=========================== -Documentation contributions -=========================== - -.. important:: - - |draft| - - |purpose| - - -[The main page for documentation contributors.] - -[We'll include docs-focused content from the :ref:`main devguide page <devguide-main>`: Quick -reference, Quick links, and so on.] - - -.. toctree:: - :maxdepth: 5 - - start-documenting - help-documenting - style-guide - markup - pull-request-lifecycle - translating - devguide diff --git a/contrib/doc/markup.rst b/contrib/doc/markup.rst deleted file mode 100644 index 96b9faad5e..0000000000 --- a/contrib/doc/markup.rst +++ /dev/null @@ -1,12 +0,0 @@ -======================= -reStructuredText markup -======================= - -.. important:: - - |draft| - - |purpose| - - -[This is the existing :ref:`markup` page from the devguide.] diff --git a/contrib/doc/pull-request-lifecycle.rst b/contrib/doc/pull-request-lifecycle.rst deleted file mode 100644 index a62e637283..0000000000 --- a/contrib/doc/pull-request-lifecycle.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _docs-pull-request-lifecycle: - -====================== -Pull request lifecycle -====================== - -.. important:: - - |draft| - - |purpose| - - -[Details of pull requests for documentation contributions. The existing -:ref:`pull-request-lifecycle` page is long and includes many details. -Some only apply to code contributions, but many are common to all -contributions. Should we keep a common page, with documentation tweaks here, or -should this page have only the documentation details even if they are duplicated -elsewhere?] - -[See :ref:`code-pull-request-lifecycle` for the code half of this conundrum.] diff --git a/contrib/doc/start-documenting.rst b/contrib/doc/start-documenting.rst deleted file mode 100644 index c5cf96161b..0000000000 --- a/contrib/doc/start-documenting.rst +++ /dev/null @@ -1,12 +0,0 @@ -=============== -Getting started -=============== - -.. important:: - - |draft| - - |purpose| - - -[This is the existing documentation :ref:`start-documenting` page from the devguide.] diff --git a/contrib/doc/style-guide.rst b/contrib/doc/style-guide.rst deleted file mode 100644 index 87762f3e03..0000000000 --- a/contrib/doc/style-guide.rst +++ /dev/null @@ -1,12 +0,0 @@ -=========== -Style guide -=========== - -.. important:: - - |draft| - - |purpose| - - -[This is the existing documentation :ref:`style-guide` page from the devguide.] diff --git a/contrib/doc/translating.rst b/contrib/doc/translating.rst deleted file mode 100644 index baface2f0d..0000000000 --- a/contrib/doc/translating.rst +++ /dev/null @@ -1,12 +0,0 @@ -=========== -Translating -=========== - -.. important:: - - |draft| - - |purpose| - - -[This is the existing :ref:`translating` page from the devguide.] diff --git a/contrib/index.rst b/contrib/index.rst deleted file mode 100644 index e519c7dab4..0000000000 --- a/contrib/index.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. _c_root: - -================================== -Python Contributor's Guide (draft) -================================== - -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> - - -.. important:: - - |draft| - - |purpose| - - -[Open question: how to divide content between this Introduction and the -:ref:`introduction <c_intro>`?] - -This guide is a comprehensive resource for :ref:`contributing <contributing>` -to Python_ -- for both new and experienced contributors. It is :ref:`maintained -<devguide>` by the same community that maintains Python. We welcome your -contributions to Python! - -We encourage everyone to contribute to Python. This guide should have -everything you need to get started and be productive. If you still have -questions after reviewing the material in this guide, the `Core Python -Mentorship`_ group is available to help you through the process. - -There are a number of ways to contribute including code, documentation, and -triaging issues. We've organized this guide to provide specifics based on the -type of activity you'll be engaged in. - - -Using this guide -================ - -We recommend reading this guide as needed. You can stop where you feel -comfortable and begin contributing immediately without reading and -understanding everything. If you do choose to skip around this guide, be aware -that it is written assuming preceding sections have been read so you may need -to backtrack to fill in missing concepts and terminology. - -No matter what kind of contribution you'll be making, you should start with -these common sections: - -* :ref:`c_intro` -* :ref:`c_project` - -Then choose a path based on your type of activity: - -*[The original table on the devguide home had a fourth column for Core -Developers. That made the table wider and more confusing. I don't think core -team members need a quick intro path since they will have been through the -devguide before.]* - -*[I haven't adjusted the links in the table yet other than to add a link to the -major section at the top of each column.]* - -.. list-table:: - :widths: 10 10 10 - :header-rows: 1 - - * - :ref:`Documentation <c_docs>` - - :ref:`Code <c_code>` - - :ref:`Triaging <c_triage>` - * - - * :ref:`docquality` - * :ref:`documenting` - * :ref:`style-guide` - * :ref:`rst-primer` - * :ref:`translating` - * :ref:`devguide` - - - * :ref:`setup` - * :ref:`help` - * :ref:`pullrequest` - * :ref:`runtests` - * :ref:`fixingissues` - * :ref:`communication` - * :ref:`gitbootcamp` - * :ref:`devcycle` - - - * :ref:`tracker` - * :ref:`triaging` - * :ref:`helptriage` - * :ref:`experts` - * :ref:`labels` - * :ref:`gh-faq` - * :ref:`triage-team` - -Core team members will find guidance in the :ref:`c_core-team` section. - -Contents -======== - -.. toctree:: - :maxdepth: 3 - - contrib-plan - intro/index - project/index - triage/index - doc/index - code/index - core-team/index - user-success - security - workflows/index - - -.. _Python: https://www.python.org/ -.. _Core Python Mentorship: https://www.python.org/dev/core-mentorship/ diff --git a/contrib/intro/index.rst b/contrib/intro/index.rst deleted file mode 100644 index 98fa30fc38..0000000000 --- a/contrib/intro/index.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _c_intro: - -============ -Introduction -============ - -.. important:: - - |draft| - - |purpose| - - - -[Open question: how to divide content between this Introduction and the -:ref:`home page <c_root>`?] - -Welcome! - -New to open source? -=================== - -Python is an open source project, with culture and techniques from the broader -open source world. You might find it helpful to read about open source in -general. A number of individuals from the Python community have contributed to -a series of excellent guides at `Open Source Guides -<https://opensource.guide/>`__. - -Anyone will find the following guides useful: - -* `How to Contribute to Open Source <https://opensource.guide/how-to-contribute/>`__ -* `Building Welcoming Communities <https://opensource.guide/building-community/>`__ - - -Healthy collaboration -===================== - -[Importance of healthy inclusive collaboration] - -[While code is a large part of the project's success, project management, documentation, governance, sprint outreach, etc. matter.] - -[We respect the individual skills people bring to the project and strive to create and maintain a culture of inclusion.] - -About this guide -================ - -Types of contribution -===================== - -[Pathways for contributors] - -Helping with this guide -======================= diff --git a/contrib/project/channels.rst b/contrib/project/channels.rst deleted file mode 100644 index 711dbe5879..0000000000 --- a/contrib/project/channels.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -====================== -Communication channels -====================== - -* Repos -* Discourse -* Discord -* Mailing lists (deprioritize) -* Where to get help diff --git a/contrib/project/conduct.rst b/contrib/project/conduct.rst deleted file mode 100644 index 37fe3bbfa7..0000000000 --- a/contrib/project/conduct.rst +++ /dev/null @@ -1,16 +0,0 @@ -=============== -Code of Conduct -=============== - -.. important:: - - |draft| - - |purpose| - - -[Brief summary of the code of conduct, with links to official source.] - -* Standard for communication -* How to report -* Enforcement details diff --git a/contrib/project/directory-structure.rst b/contrib/project/directory-structure.rst deleted file mode 100644 index 0cebb25f7f..0000000000 --- a/contrib/project/directory-structure.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_directory_structure: - -=================== -Directory structure -=================== - -[This is the :ref:`build_directory_structure` section from the devguide.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_directory_structure_start - :end-before: c_directory_structure_end diff --git a/contrib/project/generative-ai.rst b/contrib/project/generative-ai.rst deleted file mode 100644 index 6cb5b62ffc..0000000000 --- a/contrib/project/generative-ai.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -[This is the existing :ref:`generative-ai` page from the devguide.] - -.. include:: ../../getting-started/generative-ai.rst diff --git a/contrib/project/github.rst b/contrib/project/github.rst deleted file mode 100644 index fe45c6b8b1..0000000000 --- a/contrib/project/github.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -====== -GitHub -====== - -[Where are the actual artifacts?] - -* Main CPython repos -* Core workflow repos -* Infrastructure repos diff --git a/contrib/project/governance.rst b/contrib/project/governance.rst deleted file mode 100644 index a4bc66ff1b..0000000000 --- a/contrib/project/governance.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -========== -Governance -========== - -[How decisions are made, who is involved, how to participate.] - -Steering Council -================ - -Documentation Editorial Board -============================= - -Typing Council -============== - - -Others? -======= diff --git a/contrib/project/index.rst b/contrib/project/index.rst deleted file mode 100644 index 9d3c89b9a9..0000000000 --- a/contrib/project/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _c_project: - -=================== -The CPython project -=================== - -.. important:: - - |draft| - - |purpose| - - -[Give the reader an understanding of the project as a whole. What are the -moving parts, who is involved, how do they interact?] - -* Structure - -.. toctree:: - :maxdepth: 5 - - conduct - roles - governance - generative-ai.rst - github - directory-structure.rst - channels - outreach diff --git a/contrib/project/outreach.rst b/contrib/project/outreach.rst deleted file mode 100644 index d43aa8e9de..0000000000 --- a/contrib/project/outreach.rst +++ /dev/null @@ -1,12 +0,0 @@ -======== -Outreach -======== - -.. important:: - - |draft| - - |purpose| - - -* Sprints diff --git a/contrib/project/roles.rst b/contrib/project/roles.rst deleted file mode 100644 index 8336fe4651..0000000000 --- a/contrib/project/roles.rst +++ /dev/null @@ -1,17 +0,0 @@ -===== -Roles -===== - -.. important:: - - |draft| - - |purpose| - - -[Quick overview of the roles people play. Core team has its own section.] - -* Core team -* Triager -* Contributors - * types of contributions diff --git a/contrib/security.rst b/contrib/security.rst deleted file mode 100644 index db40b4a167..0000000000 --- a/contrib/security.rst +++ /dev/null @@ -1,13 +0,0 @@ -========================================= -Security and infrastructure contributions -========================================= - -.. important:: - - |draft| - - |purpose| - -* Security -* Infrastructure -* Core workflow diff --git a/contrib/triage/index.rst b/contrib/triage/index.rst deleted file mode 100644 index 0a547d9d77..0000000000 --- a/contrib/triage/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _c_triage: - -=================== -Issues and triaging -=================== - -.. toctree:: - :maxdepth: 5 - - issue-tracker - triaging - labels - reviewing - triage-team diff --git a/contrib/triage/issue-tracker.rst b/contrib/triage/issue-tracker.rst deleted file mode 100644 index a5777bc81d..0000000000 --- a/contrib/triage/issue-tracker.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`issue-tracker` page from the devguide] - -.. include:: ../../triage/issue-tracker.rst diff --git a/contrib/triage/labels.rst b/contrib/triage/labels.rst deleted file mode 100644 index c364817333..0000000000 --- a/contrib/triage/labels.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`labels` page from the devguide] - -.. include:: ../../triage/labels.rst diff --git a/contrib/triage/reviewing.rst b/contrib/triage/reviewing.rst deleted file mode 100644 index 060f6b78dd..0000000000 --- a/contrib/triage/reviewing.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. important:: - - |draft| - - |purpose| - - -========= -Reviewing -========= - -* How? Etiquette? -* How to request a review? diff --git a/contrib/triage/triage-team.rst b/contrib/triage/triage-team.rst deleted file mode 100644 index a9b59056a9..0000000000 --- a/contrib/triage/triage-team.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`triage-team` page from the devguide] - -.. include:: ../../triage/triage-team.rst diff --git a/contrib/triage/triaging.rst b/contrib/triage/triaging.rst deleted file mode 100644 index 22e1ccc657..0000000000 --- a/contrib/triage/triaging.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -[This is the existing :ref:`triaging` page from the devguide] - -.. include:: ../../triage/triaging.rst diff --git a/contrib/user-success.rst b/contrib/user-success.rst deleted file mode 100644 index 2a9ef5d4e5..0000000000 --- a/contrib/user-success.rst +++ /dev/null @@ -1,14 +0,0 @@ -======================================= -Accessibility, design, and user success -======================================= - -.. important:: - - |draft| - - |purpose| - - -* Accessibility -* Design -* User success diff --git a/contrib/workflows/codespaces.rst b/contrib/workflows/codespaces.rst deleted file mode 100644 index eb97ef7c2a..0000000000 --- a/contrib/workflows/codespaces.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_using_codespaces: - -======================= -Using GitHub Codespaces -======================= - -[This is the :ref:`using-codespaces` section from the devguide.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_codespaces_start - :end-before: c_codespaces_end diff --git a/contrib/workflows/compile.rst b/contrib/workflows/compile.rst deleted file mode 100644 index 18157b717e..0000000000 --- a/contrib/workflows/compile.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_compiling: - -================= -Compile and build -================= - -.. note:: - [This is the :ref:`compiling` section from the devguide. I think this page - is too long and could be split by build target, but we can leave that for a - later time.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_compile_and_build_start - :end-before: c_compile_and_build_end - -.. include:: ../../links.rst diff --git a/contrib/workflows/get-source.rst b/contrib/workflows/get-source.rst deleted file mode 100644 index ed56fe4e12..0000000000 --- a/contrib/workflows/get-source.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_checkout: - -=================== -Get the source code -=================== - -.. note:: - [This is the :ref:`checkout` section from the devguide. We might need to edit - it to clarify that some steps are only needed for code contribution.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_get_source_code_start - :end-before: c_get_source_code_end diff --git a/contrib/workflows/index.rst b/contrib/workflows/index.rst deleted file mode 100644 index 2c6ccf2bc9..0000000000 --- a/contrib/workflows/index.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _c_workflows: - -========= -Workflows -========= - -.. important:: - - |draft| - - |purpose| - - -This section contains details of workflows needed for all kinds of -contribution. - -.. toctree:: - - install-git.rst - get-source.rst - install-dependencies.rst - compile.rst - regenerate.rst - troubleshooting.rst - codespaces.rst diff --git a/contrib/workflows/install-dependencies.rst b/contrib/workflows/install-dependencies.rst deleted file mode 100644 index 9a511c6da2..0000000000 --- a/contrib/workflows/install-dependencies.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_build-dependencies: - -==================== -Install Dependencies -==================== - -[This is the :ref:`build-dependencies` section from the devguide.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_install_dependencies_start - :end-before: c_install_dependencies_end diff --git a/contrib/workflows/install-git.rst b/contrib/workflows/install-git.rst deleted file mode 100644 index e3d738b2ac..0000000000 --- a/contrib/workflows/install-git.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_vcsetup: - -=========== -Install Git -=========== - -[This is the :ref:`vcsetup` section from the devguide.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_install_git_start - :end-before: c_install_git_end diff --git a/contrib/workflows/regenerate.rst b/contrib/workflows/regenerate.rst deleted file mode 100644 index b5bca7dca7..0000000000 --- a/contrib/workflows/regenerate.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_regenerating: - -=============================== -Regenerating auto-created files -=============================== - -.. note:: - [These are two similar sections from the is the :ref:`setup-building` section from the devguide.] - -Regenerate ``configure`` -======================== - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_regenerate_configure_start - :end-before: c_regenerate_configure_end - -Regenerate the ABI dump -======================= - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_regenerate_abi_start - :end-before: c_regenerate_abi_end diff --git a/contrib/workflows/troubleshooting.rst b/contrib/workflows/troubleshooting.rst deleted file mode 100644 index 68aa801584..0000000000 --- a/contrib/workflows/troubleshooting.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. important:: - - |draft| - - |purpose| - -.. _c_build_troubleshooting: - -=========== -Install Git -=========== - -[This is the :ref:`build_troubleshooting` section from the devguide.] - -.. include:: ../../getting-started/setup-building.rst - :start-after: c_build_troubleshooting_start - :end-before: c_build_troubleshooting_end diff --git a/core-team/committing.rst b/core-team/committing.rst index a88ff12053..1209d6ae1e 100644 --- a/core-team/committing.rst +++ b/core-team/committing.rst @@ -35,7 +35,7 @@ to enter the public source tree. Ask yourself the following questions: Check :ref:`pull-request-lifecycle` and :ref:`helptriage` to review what is expected of a pull request. -* **Does the change break backwards-compatibility without a strong reason?** +* **Does the change break backwards compatibility without a strong reason?** :ref:`Run the entire test suite <runtests>` to make sure that everything still passes. If there is a change to the semantics, then there needs to be a strong reason, because it will cause some peoples' code to break. @@ -73,104 +73,35 @@ to enter the public source tree. Ask yourself the following questions: significant improvements, or backwards-incompatible changes), then an entry in the ``What's New in Python`` document (in ``Doc/whatsnew/``) should be added as well. Changes that affect only documentation generally do not - require a ``NEWS`` entry. (See the following section for more information.) - -.. _news-entry: -.. _what-s-new-and-news-entries: - -Updating NEWS and What's New in Python --------------------------------------- - -Changes that require NEWS entries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most changes made to the codebase deserve an entry in :cpy-file:`Misc/NEWS.d`, -except for the following: - -* documentation changes -* test changes -* strictly internal changes with no user-visible effects -* changes that already have a ``NEWS`` entry -* reverts that have not yet been included in any formal release - (including alpha and beta releases) - -For the last two, note the following: - -#. **If a change is reverted prior to release**, then the corresponding - entry is simply removed. Otherwise, a new entry must be added noting - that the change has been reverted (for example, when a feature is released in - an alpha and then cut prior to the first beta). - -#. **If a change is a fix (or other adjustment) to an earlier unreleased - change and the original** ``NEWS`` **entry remains valid**, then no additional - entry is needed. - -Changes that require "What's New in Python" entries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If a change is particularly interesting for end users (for example, new features, -significant improvements, or backwards-incompatible changes), add an entry in -the "What's New in Python" document (in :cpy-file:`Doc/whatsnew/`) -in addition to the ``NEWS`` entry. - -In most cases, it is sufficient to reuse the wording from the ``NEWS`` entry -in the "What's New in Python" entry. - -.. note:: - - A change that needs an entry in "What's New in Python", - is very likely not suitable for inclusion in a maintenance release. - -How to add a NEWS entry -^^^^^^^^^^^^^^^^^^^^^^^ - -``NEWS`` entries go into the ``Misc/NEWS.d`` directory as individual files. The -``NEWS`` entry can be created by using `blurb-it <https://blurb-it.herokuapp.com/>`__, -or the :pypi:`blurb` tool and its ``blurb add`` command. - -If you are unable to use the tool, then you can create the ``NEWS`` entry file -manually. The ``Misc/NEWS.d`` directory contains a sub-directory named -``next``, which contains various sub-directories representing classifications -for what was affected (for example, ``Misc/NEWS.d/next/Library`` for changes relating -to the standard library). The file name itself should be in the format -``<datetime>.gh-issue-<issue-number>.<nonce>.rst``: - -* ``<datetime>`` is today's date joined with a hyphen (``-``) to your current - local time, in the ``YYYY-MM-DD-hh-mm-ss`` format (for example, ``2017-05-27-16-46-23``). -* ``<issue-number>`` is the issue number the change is for (for example, ``12345`` - for ``gh-issue-12345``). -* ``<nonce>`` is a unique string to guarantee that the file name is - unique across branches (for example, ``Yl4gI2``). It is typically six characters - long, but it can be any length of letters and numbers. Its uniqueness - can be satisfied by typing random characters on your keyboard. - -As a result, a file name can look something like -``Misc/NEWS.d/next/Library/2017-05-27-16-46-23.gh-issue-12345.Yl4gI2.rst``. - -How to write a NEWS entry -^^^^^^^^^^^^^^^^^^^^^^^^^ - -All ``NEWS`` entries end up being part of the changelog. -The changelog contains *a lot* of entries, -and its intended audience is mainly users, not the core team and contributors. -Take this into consideration when wording your ``NEWS`` entry. -Describe the user-visible effects of your change succinctly and accurately; -avoid long technical elaborations, digressions, and do not expect or require -the reader to have read the actual diff for the change. - -The contents of a ``NEWS`` file should be valid reStructuredText. An 80 character -column width should be used. There is no indentation or leading marker in the -file (for example, ``-``). There is also no need to start the entry with the issue -number since it is part of the file name. You can use -:ref:`inline markups <rest-inline-markup>` too. Here is an example of a ``NEWS`` -entry:: - - Fix warning message when :func:`os.chdir` fails inside - :func:`test.support.temp_cwd`. Contributed by Chris Jerdonek. - -The inline Sphinx roles like ``:func:`` can be used help readers -find more information. You can build HTML and verify that the -link target is appropriate by using :ref:`make html <building-using-make>`. + require a ``NEWS`` entry. + + .. seealso:: + :ref:`what-s-new-and-news-entries` + + +Merging the pull request +------------------------ + +Once the pull request is ready, you (the core team member) can merge it. +If other people have been substantially involved in the review, it can be good +to wait for their approval even if a core team member has already approved the +pull request. + +The CPython repo is configured to only accept squashes. You will squash the +pull request. + +Commit message +^^^^^^^^^^^^^^ + +GitHub defaults the squashed commit message to a combined list of all of the +individual commit messages in the pull request. Do not leave those. They often +are too noisy and provide little context, especially since devs know their +work will be eventually squashed, so intermediate commit messages while +working on the pull request are not interesting. + +If you think it is important, you can summarize the collaborative work that +went into the pull request, but it is not necessary. The pull request and/or +original issue are still available for detailed investigations of history. Working with Git_ diff --git a/core-team/core-team.csv b/core-team/core-team.csv index 8c46254931..b7f25e1618 100644 --- a/core-team/core-team.csv +++ b/core-team/core-team.csv @@ -1,9 +1,12 @@ +Chris Eibl,chris-eibl,2026-05-10,, +Stan Ulbrych,StanFromIreland,2026-03-26,, +Itamar Oren,itamaro,2026-02-16,, Emma Smith,emmatyping,2025-07-31,, Tomas Roun,tomasr8,2025-06-16,, Peter Bierma,ZeroIntensity,2025-06-16,, Diego Russo,diegorusso,2025-05-13,, Bénédikt Tran,picnixz,2025-01-10,, -Savannah Ostrowski,savannahostrowski,2024-11-13,, +Savannah Ostrowski,savannahostrowski,2024-11-13,, Matt Page,mpage,2024-10-10,, Kirill Podoprigora,Eclips4,2024-09-20,, Ned Batchelder,nedbat,2024-07-16,, diff --git a/core-team/experts.rst b/core-team/experts.rst index 01935106ba..a5fd9c9358 100644 --- a/core-team/experts.rst +++ b/core-team/experts.rst @@ -20,12 +20,9 @@ following is added to an issue or pull request, they will be notified automatically. The :cpy-file:`.github/CODEOWNERS` file is also used to indicate maintainers that will be automatically added as reviewers to pull requests. +Names followed by a '*' may be assigned issues involving the module or topic. Unless a name is followed by a '*', you should never assign an issue to -that person. Names followed by a '*' may be assigned issues involving the -module or topic. - -Names followed by a '^' indicate old bugs.python.org usernames, for people -that did not transition to GitHub. +that person. The Platform and Interest Area tables list broader fields in which various people have expertise. These people can also be contacted for help, @@ -52,206 +49,115 @@ Stdlib ==================== ============================================= Module Maintainers ==================== ============================================= -__future__ __main__ gvanrossum, ncoghlan -_thread -abc annotationlib JelleZijlstra* argparse savannahostrowski*, serhiy-storchaka* -array ast benjaminp, pablogsal, isidentical, JelleZijlstra, eclips4 -asyncio 1st1, asvetlov, gvanrossum, graingert, kumaraditya303, willingc -atexit -base64 -bdb -binascii +asyncio 1st1, asvetlov, gvanrossum, graingert, kumaraditya303 bisect rhettinger* -builtins -calendar -cmath -cmd -code -codecs malemburg, doerwalter -codeop +codecs malemburg collections rhettinger* -collections.abc rhettinger*, stutzbach^ -colorsys +collections.abc rhettinger* compileall carljm -compression.bz2 -compression.gzip -compression.lzma compression.zlib Yhg1s, gpshead*, emmatyping compression.zstd emmatyping* -concurrent.futures pitrou, brianquinlan, gpshead* +concurrent.futures pitrou, gpshead* configparser ambv* contextlib ncoghlan, 1st1 -contextvars -copy avassalotti, serhiy-storchaka* -copyreg avassalotti, serhiy-storchaka* -cProfile -csv smontanaro (inactive), serhiy-storchaka* -ctypes theller (inactive), abalkin, amauryfa, meadori +copy serhiy-storchaka* +copyreg serhiy-storchaka* +csv serhiy-storchaka* curses Yhg1s dataclasses ericvsmith*, carljm -datetime abalkin, pganssle -dbm +datetime pganssle, StanFromIreland decimal facundobatista, rhettinger -difflib tim-one (inactive) dis 1st1 -doctest tim-one (inactive) -email warsaw, bitdancer*, maxking +email warsaw, bitdancer* encodings malemburg ensurepip ncoghlan, dstufft, pradyunsg, pfmoore -enum eliben*, warsaw, ethanfurman* +enum warsaw, ethanfurman* errno Yhg1s faulthandler vstinner, gpshead, ZeroIntensity* fcntl Yhg1s -filecmp -fileinput fnmatch serhiy-storchaka* -fractions ftplib giampaolo* functools rhettinger* gc pitrou, pablogsal, nascheme getopt serhiy-storchaka* -getpath FFY00 -getpass +getpath FFY00* gettext tomasr8 glob serhiy-storchaka* -grp hashlib tiran, gpshead*, picnixz -heapq rhettinger*, stutzbach^ +heapq rhettinger* hmac tiran, gpshead*, picnixz html ezio-melotti* -http -idlelib kbkaiser (inactive), terryjreedy*, serwy (inactive), - taleinat -imaplib -importlib brettcannon +idlelib terryjreedy*, taleinat +importlib brettcannon, FFY00* inspect 1st1 -io benjaminp, stutzbach^ -ipaddress pmoody^ +io benjaminp itertools rhettinger* -json etrepum (inactive), ezio-melotti, rhettinger, - serhiy-storchaka* -keyword -libmpdec -linecache +json ezio-melotti, rhettinger, serhiy-storchaka* locale malemburg logging vsajip -mailbox -marshal -math rhettinger, stutzbach^ -mimetypes +math rhettinger mmap Yhg1s -modulefinder theller (inactive), jvr^ -msvcrt -multiprocessing applio*, pitrou, jnoller^ (inactive), sbt^ (inactive), gpshead* -netrc -numbers -operator -optparse mitsuhiko, serhiy-storchaka* -os +modulefinder FFY00 +multiprocessing pitrou, gpshead* +optparse serhiy-storchaka* os.path serhiy-storchaka* parser pablogsal pathlib barneygale* pdb gaogaotiantian -pickle avassalotti, serhiy-storchaka* -pickletools avassalotti, serhiy-storchaka* -pkgutil +pickle serhiy-storchaka* +pickletools serhiy-storchaka* +pkgutil FFY00* platform malemburg -plistlib -poplib posix larryhastings, gpshead -pprint freddrake -profile -pstats pty Yhg1s* -pwd py_compile carljm pyclbr isidentical pydoc AA-Turner, serhiy-storchaka* queue rhettinger* -quopri random rhettinger re ezio-melotti, serhiy-storchaka* readline Yhg1s -reprlib resource Yhg1s -rlcompleter -runpy ncoghlan -sched -secrets -select -selectors neologix^, giampaolo -shelve -shlex -shutil tarekziade, giampaolo +runpy ncoghlan, FFY00 +selectors giampaolo +shutil giampaolo signal gpshead -site -smtplib +site FFY00* socket gpshead -socketserver -sqlite3 ghaering^, erlend-aasland* -ssl jackjansen, tiran, dstufft, alex +sqlite3 erlend-aasland* +ssl tiran, dstufft, alex stat tiran -statistics stevendaprano, rhettinger -string -stringprep -struct meadori -subprocess astrand^ (inactive), giampaolo, gpshead* +statistics rhettinger +subprocess giampaolo, gpshead* symtable benjaminp -sys -sysconfig FFY00 -syslog jafo^* -tabnanny tim-one (inactive) -tarfile gustaebel +sysconfig FFY00* tempfile serhiy-storchaka* termios Yhg1s test ezio-melotti, serhiy-storchaka* -textwrap threading pitrou, gpshead -time abalkin, pganssle -timeit -tkinter gpolo^, serhiy-storchaka* -token -tokenize meadori +time pganssle, StanFromIreland +tkinter serhiy-storchaka* tomllib hauntsaninja* -trace abalkin traceback iritkatriel tracemalloc vstinner tty Yhg1s* -turtle gregorlingl^, willingc turtledemo terryjreedy* types 1st1 typing gvanrossum, JelleZijlstra*, AlexWaygood*, carljm, sobolevn* unicodedata malemburg, ezio-melotti -unittest ezio-melotti, rbtcollins, gpshead, serhiy-storchaka* -unittest.mock +unittest ezio-melotti, gpshead, serhiy-storchaka* urllib orsenthil -uuid -venv vsajip, FFY00 -warnings -wave -weakref freddrake, nascheme -webbrowser -winreg stutzbach^ -winsound -wsgiref pjenvey -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree eliben*, scoder -xml.parsers.expat -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc +venv vsajip, FFY00* +weakref nascheme +xml.etree scoder zipapp pfmoore -zipfile alanmcintyre^, serhiy-storchaka, Yhg1s, gpshead +zipfile serhiy-storchaka, Yhg1s, gpshead zipimport Yhg1s* -zoneinfo pganssle +zoneinfo pganssle, StanFromIreland ==================== ============================================= @@ -282,18 +188,11 @@ for “their” platform as a third-party project. =================== =========== Platform Maintainers =================== =========== -AIX edelsohn, ayappanec +AIX ayappanec Android mhsmith -Cygwin jlt63^, stutzbach^ Emscripten hoodmane, pmp-p, rdb, rth, ryanking13 -FreeBSD -HP-UX iOS freakboy3742, ned-deily -JVM/Java frank.wierzbicki^ -Linux macOS ronaldoussoren, ned-deily, freakboy3742 -NetBSD1 -OS2/EMX aimacintyre^ Solaris/OpenIndiana jcea, kulikjak Windows tjguk, zooba, pfmoore =================== =========== @@ -303,7 +202,7 @@ Miscellaneous ============= ================== ========================================================== -Interest Area Maintainers +Interest area Maintainers ================== ========================================================== algorithms rhettinger*, serhiy-storchaka argument clinic larryhastings, AlexWaygood*, erlend-aasland, @@ -316,51 +215,249 @@ bytecode benjaminp, 1st1, markshannon, brandtbucher, carljm, iritkatr context managers ncoghlan core workflow Mariatta, ezio-melotti, hugovk, AA-Turner cryptography gpshead, dstufft, picnixz -data formats database malemburg -devguide merwok, ezio-melotti, willingc, Mariatta, hugovk, - AA-Turner -documentation ezio-melotti, merwok, JulienPalard, willingc, hugovk, - AA-Turner, AlexWaygood* +devguide merwok, ezio-melotti, Mariatta, hugovk, + AA-Turner, StanFromIreland +documentation ezio-melotti, merwok, JulienPalard, hugovk, + AA-Turner, AlexWaygood*, StanFromIreland emoji Mariatta -extension modules encukou, ncoghlan -filesystem giampaolo -frozen modules ericsnowcurrently, gvanrossum, kumaraditya303 +extension modules encukou, ncoghlan, FFY00 +frozen modules ericsnowcurrently, gvanrossum, kumaraditya303, FFY00 f-strings ericvsmith* -GUI -i18n malemburg, merwok, tomasr8 -import machinery brettcannon, ncoghlan, ericsnowcurrently, FFY00 -initialization FFY00 -io benjaminp, stutzbach^, gpshead +i18n malemburg, merwok, tomasr8, StanFromIreland +import machinery brettcannon, ncoghlan, ericsnowcurrently, FFY00* +initialization FFY00* +io benjaminp, gpshead JIT brandtbucher*, savannahostrowski* locale malemburg -mathematics malemburg, stutzbach^, rhettinger, serhiy-storchaka +mathematics malemburg, rhettinger, serhiy-storchaka memory management tim-one, malemburg, Yhg1s, nascheme -memoryview -networking giampaolo, gpshead +networking gpshead object model benjaminp, Yhg1s -packaging tarekziade, malemburg, alexis^, merwok, dstufft, pfmoore, emmatyping +packaging malemburg, merwok, dstufft, pfmoore, emmatyping, FFY00 pattern matching brandtbucher* PEG parser gvanrossum, pablogsal, lysnikolaou performance vstinner, serhiy-storchaka*, 1st1, rhettinger, markshannon, brandtbucher, carljm, Fidget-Spinner, AlexWaygood*, nascheme -pip ncoghlan, dstufft, pfmoore, Marcus.Smith^, pradyunsg -release management tarekziade, malemburg, benjaminp, warsaw, - gvanrossum, anthonybaxter^, merwok, ned-deily, - birkenfeld, JulienPalard, hugovk -runtime lifecycle ericsnowcurrently, kumaraditya303, zooba, ZeroIntensity, nascheme +pip ncoghlan, dstufft, pfmoore, pradyunsg +release management malemburg, benjaminp, warsaw, + gvanrossum, merwok, ned-deily, + JulienPalard, hugovk +runtime lifecycle ericsnowcurrently, kumaraditya303, zooba, ZeroIntensity, nascheme, FFY00 str.format ericvsmith* subinterpreters ericsnowcurrently, kumaraditya303, ZeroIntensity* symbol table JelleZijlstra, carljm testing ezio-melotti -test coverage threads gpshead -time and dates malemburg, abalkin, pganssle +time and dates malemburg, pganssle, StanFromIreland Unicode malemburg, ezio-melotti, benjaminp version control merwok, ezio-melotti ================== ========================================================== +Experts emeritus +================ + +Names followed by a '^' indicate old bugs.python.org usernames, for people +that did not transition to GitHub. + +Stdlib +------ + +.. list-table:: + + * - Module + - Expert + + * - codecs + - doerwalter + + * - collections.abc + - stutzbach^ + + * - concurrent.futures + - brianquinlan + + * - copy + - avassalotti + + * - copyreg + - avassalotti + + * - csv + - smontanaro + + * - ctypes + - theller, abalkin, amauryfa, meadori + + * - datetime + - abalkin + + * - difflib + - tim-one + + * - doctest + - tim-one + + * - email + - maxking + + * - enum + - eliben + + * - heapq + - stutzbach + + * - idlelib + - kbkaiser, serwy + + * - io + - stutzbach^ + + * - ipaddress + - pmoody^ + + * - json + - etrepum + + * - math + - stutzbach^ + + * - modulefinder + - theller, jvr^ + + * - multiprocessing + - applio, jnoller^, sbt^ + + * - optparse + - mitsuhiko + + * - pickle + - avassalotti + + * - pickletools + - avassalotti + + * - pprint + - freddrake + + * - selectors + - neologix^ + + * - shutil + - tarekziade + + * - sqlite3 + - ghaering^ + + * - ssl + - jackjansen + + * - statistics + - stevendaprano + + * - struct + - meadori + + * - subprocess + - astrand^ + + * - syslog + - jafo^ + + * - tabnanny + - tim-one + + * - tarfile + - gustaebel + + * - time + - abalkin + + * - tkinter + - gpolo^ + + * - tokenize + - meadori + + * - trace + - abalkin + + * - turtle + - gregorlingl^ + + * - unittest + - rbtcollins + + * - weakref + - freddrake + + * - winreg + - stutzbach^ + + * - wsgiref + - pjenvey + + * - xml.etree + - eliben + + * - zipfile + - alanmcintyre^ + +Platforms +--------- + +.. list-table:: + + * - Platform + - Expert + + * - AIX + - edelsohn + + * - Cygwin + - jlt63^, stutzbach^ + + * - JVM/Java + - frank.wierzbicki^ + + * - OS2/EMX + - aimacintyre^ + + +Miscellaneous +------------- + +.. list-table:: + + * - Interest Area + - Expert + + * - filesystem + - giampaolo + + * - io + - stutzbach^ + + * - mathematics + - stutzbach^ + + * - networking + - giampaolo + + * - packaging + - tarekziade, alexis^ + + * - pip + - Marcus.Smith^ + + * - release management + - tarekziade, anthonybaxter^, birkenfeld + + * - time and dates + - abalkin + + Documentation translations ========================== diff --git a/core-team/motivations.rst b/core-team/motivations.rst index d0e5a0cc18..7babb5f7da 100644 --- a/core-team/motivations.rst +++ b/core-team/motivations.rst @@ -115,7 +115,7 @@ participating in the CPython core development process: As a core team member, she is primarily interested in helping to ensure Python's continued suitability for educational, testing and data analysis use cases, as well as in encouraging good architectural practices when assembling Python - applications and test harnesses from open source components. + applications and test harnesses from open-source components. Note: prior to August 2023, Alyssa used her birth name (Nick Coghlan). Some records (for example, mailing list archives, version control history) will still reference that name. @@ -175,14 +175,14 @@ participating in the CPython core development process: available, working on the infrastructure that supports CPython development, specifically the Roundup-based bug tracker and the buildbot system. - David currently does both proprietary and open source development work, + David currently does both proprietary and open-source development work, primarily in Python, through the company in which he is a partner, `Murray & Walker, Inc <https://www.murrayandwalker.com>`__. He has done contract work focused specifically on CPython development both through the PSF (the kickstart of the email Unicode API development) and directly funded by interested corporations (additional development work on email funded by QNX, and work on CPython ICC support funded by Intel). He would like to - spend more of his (and his company's) time on open source work, and so is + spend more of his (and his company's) time on open-source work, and so is actively seeking additional such contract opportunities. .. topic:: Antoine Pitrou (France) @@ -202,7 +202,7 @@ participating in the CPython core development process: concurrent programming. As a professional, Antoine has been first specializing in network - programming, and more lately in open source data science infrastructure. + programming, and more lately in open-source data science infrastructure. He has made numerous contributions to Numba, Dask and is currently working full time on Apache Arrow as a technical leader at QuantStack. @@ -269,7 +269,7 @@ participating in the CPython core development process: Carol is focused on Python's usage in education and scientific research. She is interested in distributed computing, organizational development, - operational workflows, and sustainability of open source projects. + operational workflows, and sustainability of open-source projects. .. _goals-of-the-motivations-page: @@ -282,7 +282,7 @@ strongly suggest that the current core development process is bottlenecked on core team time. This is most clearly indicated in the first metrics graph, which shows both the number of open issues and the number of pull requests awaiting review growing steadily over time, despite CPython being one of the most -active open source projects in the world. This bottleneck then impacts not only +active open-source projects in the world. This bottleneck then impacts not only resolving open issues and accepting submitted pull requests, but also the process of identifying, nominating and mentoring new core team members. diff --git a/core-team/responsibilities.rst b/core-team/responsibilities.rst index 9f5c62b728..130e763d56 100644 --- a/core-team/responsibilities.rst +++ b/core-team/responsibilities.rst @@ -123,7 +123,7 @@ to better assess the sustainability of current contributions to CPython core development, and also serves as a referral list for organisations seeking commercial Python support from the core development community. -And finally, enjoy yourself! Contributing to open source software should be fun +And finally, enjoy yourself! Contributing to open-source software should be fun (overall). If you find yourself no longer enjoying the work then either take a break or figure out what you need to do to make it enjoyable again. diff --git a/developer-workflow/c-api.rst b/developer-workflow/c-api.rst index e7f129a27d..427d9fd96a 100644 --- a/developer-workflow/c-api.rst +++ b/developer-workflow/c-api.rst @@ -395,9 +395,8 @@ If following these guidelines would hurt performance, add a fast function API. If anything is unclear, or you have a good reason to break the guidelines, -consider discussing the change at the `capi-sig`_ mailing list. - -.. _capi-sig: https://mail.python.org/mailman3/lists/capi-sig.python.org/ +consider discussing the change in the `C API category +<https://discuss.python.org/c/core-dev/c-api/>`__ on Discourse. Adding a new definition to the Limited API ------------------------------------------ diff --git a/developer-workflow/communication-channels.rst b/developer-workflow/communication-channels.rst index 43c00b9e1b..101bb06122 100644 --- a/developer-workflow/communication-channels.rst +++ b/developer-workflow/communication-channels.rst @@ -22,6 +22,38 @@ in return. .. _Diversity Statement: https://www.python.org/psf/diversity/ +.. _multicultural-communication: + +Communicating across cultures and languages +=========================================== + +Not every contributor writes English as a first language, and phrasing that +feels neutral in one culture can read as blunt or rude in another. + +**Practice active listening.** Try to focus on understanding the message before reacting. + +**Confirm understanding.** Ask open-ended questions and paraphrase to avoid +misunderstandings. If you're unsure what someone meant, ask: "I want to make +sure I understand: are you saying X?" + +**Use translation tools freely.** If English isn't your first language, +translation software or AI tools may help you check that your message has the +tone you intend. Getting the phrasing wrong can make a reasonable point sound +more aggressive than it is. + +**Watch out for rhetorical questions.** They read as hostile in text even when +that's not the intent at all: + +* Instead of "Why do you think it is wrong?!", try "That doesn't sound right + to me. Here's why: …" +* Instead of "Did you even read the docs?", try "The relevant documentation + is at …" + +**Be patient with newcomers.** If someone doesn't know how things work here, +point them to the right docs or give them the context they need. Criticism +without guidance isn't helpful. + + .. _mailinglists: Mailing lists @@ -51,7 +83,7 @@ Mailing lists .. _python-committers: https://mail.python.org/mailman3/lists/python-committers.python.org/ .. _python-dev: https://mail.python.org/mailman3/lists/python-dev.python.org/ - .. _python-ideas: https://mail.python.org/mailman3/lists/python-ideas.python.org + .. _python-ideas: https://mail.python.org/archives/list/python-ideas@python.org/ General Python questions should go to `python-list`_ or `tutor`_ or similar resources, such as `Stack Overflow`_ or the ``#python`` IRC channel @@ -62,13 +94,12 @@ issue tracker is the place to discuss and work on improvements to the CPython core development workflow. A complete list of Python mailing lists can be found at -https://mail.python.org/mailman/listinfo (older lists, using Mailman2) or -https://mail.python.org/mailman3/ (newer lists, using Mailman3). Some lists may also +https://mail.python.org/mailman3/. Some lists may also be mirrored at `GMANE <https://gmane.io/>`__ and can be read and posted to in various ways, including via web browsers, NNTP newsreaders, and RSS feed readers. -.. _python-list: https://mail.python.org/mailman/listinfo/python-list -.. _tutor: https://mail.python.org/mailman/listinfo/tutor +.. _python-list: https://mail.python.org/mailman3/lists/python-list.python.org/ +.. _tutor: https://mail.python.org/mailman3/lists/tutor.python.org/ .. _Stack Overflow: https://stackoverflow.com/ .. _Libera.Chat: https://libera.chat/ @@ -239,7 +270,7 @@ that way. You can find their blogs (and various other developers who use Python) at `Planet Python <https://planetpython.org/>`__. -Setting expectations for open source participation +Setting expectations for open-source participation ================================================== Burn-out is common in open source due to a misunderstanding of what users, contributors, diff --git a/developer-workflow/development-cycle.rst b/developer-workflow/development-cycle.rst index 2f1cadbbd9..ea46c2405f 100644 --- a/developer-workflow/development-cycle.rst +++ b/developer-workflow/development-cycle.rst @@ -74,12 +74,15 @@ produced from a maintenance branch are called **bugfix** or **maintenance** releases; the terms are used interchangeably. These releases have a **micro version** number greater than zero. -The only changes allowed to occur in a maintenance branch without debate are -bug fixes, test improvements, and edits to the documentation. +Changes backported to a maintenance branch fall into two groups. *Low-risk* +changes (bug fixes, test improvements, and documentation edits) may be +backported without debate. *Higher-risk* changes (new features, semantic +changes, and performance improvements) can introduce regressions, so they are +not backported as a matter of course. Also, a general rule for maintenance branches is that compatibility must not be broken at any point between sibling micro releases (3.12.1, 3.12.2, -etc.). For both rules, only rare exceptions are accepted and **must** be -discussed first. +etc.). For both rules, only rare exceptions are accepted, and each requires a +strong case agreed upon in discussion beforehand. Backporting changes reduces the risk of future conflicts. For documentation, it increases the visibility of improvements, @@ -232,7 +235,10 @@ This includes, for example: Before adding a new repository to the organization, open a discussion to seek consensus in the `Committers Discourse category <https://discuss.python.org/c/committers/5>`__. Once people are satisfied with that, ask the `Python steering council <https://github.com/python/steering-council>`__ -to grant permission. +to grant permission. Note that this process is not necessary for +:ref:`docs translations <translation-repo>` following +:pep:`PEP 545 <545#repository-for-po-files>`, which can be added at a +core team member’s discretion. Note that several repositories remain in the organization for historic reasons, and would probably not be appropriate to add today. @@ -268,8 +274,8 @@ This role is paramount to the security of the Python Language, Community, and Infrastructure. The Executive Director of the Python Software Foundation delegates authority on -GitHub Organization Owner Status to Ee Durbin - Python Software -Foundation Director of Infrastructure. Common reasons for this role are: +GitHub Organization Owner Status to Jacob Coffee - Python Software +Foundation Infrastructure Engineer. Common reasons for this role are: Infrastructure Staff Membership, Python Software Foundation General Counsel, and Python Software Foundation Staff as fallback. @@ -293,7 +299,7 @@ Current owners +----------------------+--------------------------------+-----------------+ | Donald Stufft | Infrastructure Staff | dstufft | +----------------------+--------------------------------+-----------------+ -| Ee Durbin | PSF Director of Infrastructure | ewdurbin | +| Ee Durbin | Infrastructure Staff | ewdurbin | +----------------------+--------------------------------+-----------------+ | Jacob Coffee | PSF Infrastructure Engineer | JacobCoffee | +----------------------+--------------------------------+-----------------+ diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 4d8c0ffca1..cab46c834b 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -547,7 +547,7 @@ Now that the configuration is in place, it remains to compile the project: .. tip:: - Use ``make -jN`` to speed-up compilation by utilizing as many CPU cores + Use ``make -jN`` to speed up compilation by utilizing as many CPU cores as possible, where *N* is as many CPU cores you want to spare (and have memory for). Be careful using ``make -j`` with no argument, as this puts no limit on the number of jobs, and compilation can sometimes use up a diff --git a/developer-workflow/index.rst b/developer-workflow/index.rst index 9919398e62..e04fc8a8b3 100644 --- a/developer-workflow/index.rst +++ b/developer-workflow/index.rst @@ -15,5 +15,3 @@ Development workflow c-api grammar porting - sbom - psrt diff --git a/developer-workflow/psrt.rst b/developer-workflow/psrt.rst deleted file mode 100644 index cf5acd2b70..0000000000 --- a/developer-workflow/psrt.rst +++ /dev/null @@ -1,194 +0,0 @@ -Python Security Response Team (PSRT) -==================================== - -The Python Security Response Team (PSRT) is responsible for handling -vulnerability reports for CPython and pip. - -Vulnerability report triage ---------------------------- - -Vulnerability reports are sent to one of two locations, -the long-standing ``security@python.org`` mailing list -or using the private vulnerability reporting feature -of GitHub Security Advisories (GHSA). - -For reports sent to ``security@python.org``, a PSRT admin -will triage the report and if the report seems plausible -(that is, not spam and for the correct project) will reply with -instructions on how to report the vulnerability on GitHub. - -If the reporter doesn't want to use GitHub's Security Advisories feature -then the PSRT admins can create a draft report on behalf of the reporter. - -Coordinating a vulnerability report ------------------------------------ - -Each report will have a member of the PSRT assigned as the "coordinator". -The coordinator will be responsible for following the below process and -will be publicly credited on vulnerability records post-publication. - -If a coordinator can't complete the process for any reason (time obligation, -vacation, etc.) they must find a replacement coordinator in the PSRT -and reassign the vulnerability report appropriately. - -Coordinators are expected to collaborate with other PSRT and core team members -when needed for guidance on whether the report is an actual vulnerability, -severity, advisory text, and fixes. - -**The vulnerability coordination process is:** - -* Coordinator will determine whether the report constitutes a vulnerability. If the report isn't a vulnerability, - the reporter should be notified appropriately. Close the GHSA report, the report can be reopened if - sufficient evidence is later obtained that the report is a vulnerability. - -* After a vulnerability report is accepted, a Common Vulnerabilities and Exposures (CVE) ID must be assigned. If this is not done - automatically, then a CVE ID can be obtained by the coordinator sending an email to ``cna@python.org``. - No details about the vulnerability report need to be shared with the PSF CVE Numbering Authority (CNA) for a CVE ID to be reserved. - -* If the report is a vulnerability, the coordinator will determine the severity of the vulnerability. Severity is one of: - **Low**, **Medium**, **High**, and **Critical**. Coordinators can use their knowledge of the code, how the code is likely used, - or another mechanism like Common Vulnerability Scoring System (CVSS) for determining a severity. Add this information to the GitHub Security Advisory. - -* Once a CVE ID is assigned, the coordinator will share the acceptance and CVE ID with the reporter. - Use this CVE ID for referencing the vulnerability. The coordinator will ask the reporter - if the reporter would like to be credited publicly for the report and if so, how they would like to be credited. - Add this information to the GitHub Security Advisory. - -* The coordinator authors the vulnerability advisory text. The advisory must include the following information: - - * Title should be a brief description of the vulnerability and affected component - (for example, "Buffer over-read in SSLContext.set_npn_protocols()") - - * Short description of the vulnerability, impact, and the conditions where the affected component is vulnerable, if applicable. - - * Affected versions. This could be "all versions", but if the vulnerability exists in a new feature - or removed feature then this could be different. Include versions that are end-of-life in this calculation - (for example, "Python 3.9 and earlier", "Python 3.10 and later", "all versions of Python"). - - * Affected components and APIs. The module, function, class, or method must be specified so users can - search their codebase for usage. For issues affecting the entire project, this can be omitted. - - * Mitigations for the vulnerability beyond upgrading to a fixed version, if applicable. - - This can all be done within the GitHub Security Advisory UI for easier collaboration between reporter and coordinator. - -* The coordinator determines the fix approach and who will provide a fix. - Some reporters are willing to provide or collaborate to create a fix, - otherwise relevant core team members can be invited to collaborate by - the coordinator. - - * For **Low** and **Medium** severity vulnerabilities it is acceptable - to develop a fix in public. - The pull request must be marked with the ``security`` and ``release-blocker`` - labels so that a release is not created without including the fix. - - * For **High** and **Critical** severity vulnerabilities the fix must be - developed privately using GitHub Security Advisories' "Private Forks" feature. - Core team members can be added to the GitHub Security Advisory via "collaborators" - to work on the fix together. Once a fix is approved privately and tested, - a public issue and pull request can be created with - the ``security`` and ``release-blocker`` labels. - -* Once the pull request is merged the advisory can be published. The coordinator will send the advisory by email - to ``security-announce@python.org`` using the below template. Backport labels must be added as appropriate. - After the advisory is published a CVE record can be created. - -Handling code signing certificate reports ------------------------------------------ - -Python signs binaries using Azure Trusted Signing and Apple Developer ID -certificates. If a code signing certificate is reported as "compromised" or -"malware signed with certificate", the Python Security Response Team must -request the following information from the reporter: - -* Checksum(s) of binaries signed by certificate. -* Signature(s) of binaries signed by certificate. - -To avoid unnecessary user confusion and churn around revoking code signing -certificates, any reports **must be verifiable independently by the PSRT before -taking destructive actions**, such as revoking certificates. With this -information the PSRT can take investigative steps to verify the report, such as: - -* Downloading and checking artifacts from the associated Azure Pipelines - executions against the reported list of checksums. -* Verifying the validity of the signatures. `Past reports - <https://discuss.python.org/t/103356/2>`__ have contained signatures that - purported to be from Python code signing certificates, but were not valid. -* Checking the Azure Pipelines and Azure Trusted Signing audit logs for signs of - compromise. - -If any signs of compromise or incorrectly signed binaries are discovered by the -PSRT, only then will certificates be revoked and an advisory published. -If compromise is reported, the following non-destructive actions can be taken by -the PSRT without verifying the reported information as a precaution, if -relevant: - -* Rotating secrets associated with code signing (``TrustedSigningSecret`` for - Azure Trusted Publishing). -* Resetting passwords for accounts with access to signing certificates. - -Template responses ------------------- - -These template responses should be used as guidance for messaging -in various points in the process above. They are not required to be sent as-is, -please feel free to adapt them as needed for the current context. - -**Directing to GitHub Security Advisories:** - -.. highlight:: none - -:: - - Thanks for submitting this report. - We use GitHub Security Advisories for triaging vulnerability reports, - are you able to submit your report directly to GitHub? - - https://github.com/python/cpython/security/advisories/new - - If you're unable to submit a report to GitHub (due to not having a GitHub - account or something else) let me know and I will create a GitHub Security - Advisory on your behalf, although you won't be able to participate directly - in discussions. - -**Rejecting a vulnerability report:** - -:: - - Thanks for your report. We've determined that the report doesn't constitute - a vulnerability. Let us know if you disagree with this determination. - If you are interested in working on this further, you can optionally open a - public issue on GitHub. - -**Accepting a vulnerability report:** - -:: - - Thanks for your report. We've determined that the report - is a vulnerability. We've assigned {CVE-YYYY-XXXX} and determined - a severity of {Low,Medium,High,Critical}. Let us know if you disagree - with the determined severity. - - If you would like to be publicly credited for this vulnerability as the - reporter, please indicate that, along with how you would like to be - credited (name or organization). - - Please keep this vulnerability report private until we've published - an advisory to ``security-announce@python.org``. - -**Advisory email:** - -:: - - Title: [{CVE-YYYY-XXXX}] {title} - - There is a {LOW, MEDIUM, HIGH, CRITICAL} severity vulnerability - affecting {project}. - - {description} - - Please see the linked CVE ID for the latest information on - affected versions: - - * https://www.cve.org/CVERecord?id={CVE-YYYY-XXXX} - * {pull request URL} diff --git a/developer-workflow/stdlib.rst b/developer-workflow/stdlib.rst index b683e55e96..ec10977221 100644 --- a/developer-workflow/stdlib.rst +++ b/developer-workflow/stdlib.rst @@ -126,7 +126,7 @@ If the module you want to propose adding to the stdlib meets the requirements, you may propose its inclusion by following the :abbr:`PEP (Python Enhancement Proposal)` process. See :pep:`1` for details, -and the :pep:`PEP index <0>` for previously-accepted PEPs +and the :pep:`PEP index <0>` for previously accepted PEPs that have proposed a module for inclusion. If the PEP is accepted, then the module will be added to the stdlib diff --git a/development-tools/warnings.rst b/development-tools/warnings.rst index b30d811311..1026082528 100644 --- a/development-tools/warnings.rst +++ b/development-tools/warnings.rst @@ -29,7 +29,7 @@ What to do if a warning check fails GitHub CI --------------------------------------------- The :cpy-file:`Tools/build/check_warnings.py` tool will fail if the compiler generates -more or less warnings than expected for a given source file as defined in the +more or fewer warnings than expected for a given source file as defined in the platform-specific warning ignore file. The warning ignore file is either :cpy-file:`Tools/build/.warningignore_ubuntu` or :cpy-file:`Tools/build/.warningignore_macos` depending on the platform. @@ -44,7 +44,7 @@ If a warning check fails with: warning ignore file. If the file exists in the warning ignore file increment the count by the number of newly introduced warnings. -* Unexpected improvements (less warnings) +* Unexpected improvements (fewer warnings) * Document in the PR that the change reduces the number of compiler warnings. Decrement the count in the platform-specific warning diff --git a/documentation/devguide.rst b/documentation/devguide.rst index fc12c369f9..74f734831b 100644 --- a/documentation/devguide.rst +++ b/documentation/devguide.rst @@ -4,13 +4,7 @@ Helping with the Developer's Guide ================================== -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst .. highlight:: console diff --git a/documentation/markup.rst b/documentation/markup.rst index f9291a335b..282b80f911 100644 --- a/documentation/markup.rst +++ b/documentation/markup.rst @@ -225,7 +225,7 @@ Explicit markup --------------- "Explicit markup" is used in reST for most constructs that need special -handling, such as footnotes, specially-highlighted paragraphs, comments, and +handling, such as footnotes, specially highlighted paragraphs, comments, and generic directives. An explicit markup block begins with a line starting with ``..`` followed by @@ -349,22 +349,6 @@ they are used in the Python documentation. <https://www.sphinx-doc.org/>`__. -Meta-information markup ------------------------ - -.. describe:: sectionauthor - - Identifies the author of the current section. The argument should include - the author's name such that it can be used for presentation (though it isn't) - and email address. The domain name portion of the address should be lower - case. Example:: - - .. sectionauthor:: Guido van Rossum <guido@python.org> - - Currently, this markup isn't reflected in the output in any way, but it helps - keep track of contributions. - - Module-specific markup ---------------------- @@ -373,17 +357,11 @@ module being documented. Each module should be documented in its own file. Normally this markup appears after the title heading of that file; a typical file might start like this:: - :mod:`parrot` -- Dead parrot access - =================================== + :mod:`!parrot` -- Dead parrot access + ==================================== .. module:: parrot - :platform: Unix, Windows :synopsis: Analyze and reanimate dead parrots. - .. moduleauthor:: Eric Cleese <eric@python.invalid> - .. moduleauthor:: John Idle <john@python.invalid> - -As you can see, the module-specific markup consists of two directives, the -``module`` directive and the ``moduleauthor`` directive. .. describe:: module @@ -391,24 +369,12 @@ As you can see, the module-specific markup consists of two directives, the or submodule. The name should be fully qualified (that is, including the package name for submodules). - The ``platform`` option, if present, is a comma-separated list of the - platforms on which the module is available (if it is available on all - platforms, the option should be omitted). The keys are short identifiers; - examples that are in use include "IRIX", "Mac", "Windows", and "Unix". It is - important to use a key which has already been used when applicable. - The ``synopsis`` option should consist of one sentence describing the module's purpose -- it is currently only used in the Global Module Index. The ``deprecated`` option can be given (with no value) to mark a module as deprecated; it will be designated as such in various locations then. -.. describe:: moduleauthor - - The ``moduleauthor`` directive, which can appear multiple times, names the - authors of the module code, just like ``sectionauthor`` names the author(s) - of a piece of documentation. It too does not result in any output currently. - .. note:: It is important to make the section title of a module-describing file @@ -1057,6 +1023,17 @@ Paragraph-level markup These directives create short paragraphs and can be used inside information units as well as normal text: +.. describe:: availability + + This directive documents the platforms on which a module or feature + is available. For example:: + + .. availability:: Unix, not WASI, not Android. + + :: + + .. availability:: Linux >= 3.0 with glibc >= 2.14. + .. describe:: note An especially important bit of information about an API that a user should be @@ -1167,6 +1144,19 @@ units as well as normal text: .. deprecated-removed:: next 4.0 +.. describe:: soft-deprecated + + Indicates the version from which the described feature is :term:`soft deprecated`. + + There is one required argument: the version from which the feature is soft + deprecated. + Use the word ``next`` to indicate the API will be first soft deprecated in + the upcoming release. + + Example:: + + .. soft-deprecated:: next + .. describe:: impl-detail This directive is used to mark CPython-specific information. Use either with diff --git a/documentation/start-documenting.rst b/documentation/start-documenting.rst index 7865f125d6..917cb30bfa 100644 --- a/documentation/start-documenting.rst +++ b/documentation/start-documenting.rst @@ -5,13 +5,7 @@ Getting started =============== -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst .. highlight:: rest diff --git a/documentation/style-guide.rst b/documentation/style-guide.rst index d90a14ce90..73618f9aec 100644 --- a/documentation/style-guide.rst +++ b/documentation/style-guide.rst @@ -79,6 +79,16 @@ C API CPU Central processing unit. No need to spell out. +free-threaded + The preferred term for the build mode that makes the global interpreter + lock (GIL) optional (per :pep:`703`). Avoid using "No-GIL" to avoid + double negatives (for example, "non-no-GIL"). + +open source + Follow the usual English rules for compound words. When used as an + adjective, hyphenate: "open-source software". When used as a noun, don't use + a hypen: "open source is a collaboration model.." + POSIX The name assigned to a particular group of standards. This is always uppercase. @@ -87,7 +97,7 @@ Python The name of our favorite programming language is always capitalized. reST - For "reStructuredText," an easy to read, plaintext markup syntax + For "reStructuredText," an easy to read, plain-text markup syntax used to produce Python documentation. When spelled out, it is always one word and both forms start with a lowercase 'r'. @@ -116,6 +126,24 @@ Don't use Latin abbreviations like "e.g." or "i.e." where English words will do, such as "for example" or "that is." +Charged terminology to avoid +============================ + +Avoid terminology that may be considered insensitive or exclusionary. + +.. list-table:: + :header-rows: 1 + + * - Avoid + - Instead + * - whitelist + - allowlist + * - blacklist + - blocklist, denylist + * - master/slave + - main, parent/child, server/client, primary/secondary + + .. index:: diataxis .. _diataxis: @@ -175,18 +203,19 @@ the second mention is more appropriate for a link. Some units are long enough to have a few repeated links. Use judgement to decide when a link will help the reader. -Do not use a link when the link would point to the current unit. It's natural -to use the name of a function in the documentation for the function, but a link -on that function name that simply reloads the section the user is already -reading is useless and distracting. +We use the :pypi:`linklint Sphinx extension <linklint>` to suppress two kinds +of excessive links: references in a section to itself and duplicate references +in a paragraph. Previously, editors had to carefully use an exclamation mark +in a Sphinx reference (``:func:`!map```) to prevent such links. Do not use links in section headers. They distract from the title of the section. The term will be mentioned in the paragraph text and can be linked from there. -Sphinx provides ways to automatically add links to references, and a way to +Sphinx automatically adds links to references, and provides a way to suppress the link. Using roles like ``:func:`map``` will link to the -documentation for ``map``. You can suppress the link while keeping the +documentation for ``map``. If automatic link suppression isn't enough, +you can suppress the link while keeping the semantic presentation of the function name by adding an exclamation point prefix: ``:func:`!map```. See :ref:`roles` for more details. @@ -224,8 +253,8 @@ For new documentation, do not use a byline (naming the author of the document). Explicit attribution tends to discourage other users from updating community documentation. -Existing documentation with bylines will not be changed unless the author -decides to do so. This is subject to change in the future. +Existing bylines are for historical interest only. They do not imply ownership +or necessary approvals, and do not prevent edits or updates by others. Pronunciation of dunder names diff --git a/documentation/translations/coordinating.rst b/documentation/translations/coordinating.rst index a7872b6aec..82cfce74fa 100644 --- a/documentation/translations/coordinating.rst +++ b/documentation/translations/coordinating.rst @@ -64,6 +64,8 @@ to make your translation easier to find. In the entry you can also include links to guides or other resources for translators. +.. _translation-repo: + Repository ---------- diff --git a/documentation/translations/translating.rst b/documentation/translations/translating.rst index 2e4f6cf91c..820e2dd694 100644 --- a/documentation/translations/translating.rst +++ b/documentation/translations/translating.rst @@ -101,6 +101,11 @@ For more details about translations and their progress, see * - Swedish (sv) - Daniel Nylander (:github-user:`yeager`) - :github:`GitHub <python/python-docs-sv>` + * - Tamil (ta) + - | Murugan Santhosh (:github-user:`terminaljoint`), + | Hari (:github-user:`nammahari`) + - :github:`GitHub <Terminal-Joint/python-docs-ta>`, + `Discord <https://discord.gg/JNWjefK2yG>`__ * - `Traditional Chinese (zh-tw) <https://docs.python.org/zh-tw/>`__ - | 王威翔 Matt Wang (:github-user:`mattwang44`), | Josix Wang (:github-user:`josix`) @@ -113,6 +118,10 @@ For more details about translations and their progress, see - Dmytro Kazanzhy (:github-user:`kazanzhy`, `email <mailto:dkazanzhy@gmail.com>`__) - :github:`GitHub <python/python-docs-uk>`, `Transifex <tx_>`_ + * - Vietnamese (vi) + - Duc-Tam Nguyen (:github-user:`tamnd`) + - :github:`GitHub <tamnd/python-docs-vi>`, + `Transifex <tx_>`_ How to get help @@ -361,5 +370,51 @@ If you would like to coordinate, open a pull request in the at the top of this page, and ping ``@python/editorial-board``. +How do I merge translations back in after docs are moved? +--------------------------------------------------------- + +When docs are moved in the CPython repository, translated messages are not +moved automatically and will be lost. +They can be recovered by merging them into the new locations. +Note that this is not necessary for Transifex-based translations, as Transifex's +translation memory will automatically copy them over. + +The :pypi:`pomerge` tool merges translations between PO files by matching +messages, regardless of file paths. To use it, first install the package: + +.. code-block:: bash + + pip install pomerge + +Then, merge translations from a specific commit (replace :samp:`{COMMIT_HASH}` +with the commit hash from before the files were moved): + +.. TODO: Provide Windows instructions. + +.. tab:: Unix + + .. code-block:: bash + + # These commands are to be run in the root of the translation repo + + # Check out a commit before the move + git checkout COMMIT_HASH -- . + + # Copy translations to a temporary dir + cp -r . /tmp/old-po-files + + # Return to the current version + git checkout HEAD -- . + + # Merge translations from temporary dir back in + pomerge --from /tmp/old-po-files/**/*.po --to **/*.po --clear + + # Clean up temporary dir + rm -rf /tmp/old-po-files + +After running ``pomerge``, review the changes and commit the updated files. +You may also need to rewrap the lines (see :pypi:`powrap`). + + .. _discourse: https://discuss.python.org/c/documentation/translations/ .. _tx: https://explore.transifex.com/python-doc/python-newest/ diff --git a/getting-started/ai-tools.rst b/getting-started/ai-tools.rst new file mode 100644 index 0000000000..e7d94d3517 --- /dev/null +++ b/getting-started/ai-tools.rst @@ -0,0 +1,64 @@ +.. _ai-tools: +.. _generative-ai: + +============================= +Guidelines for using AI tools +============================= + +The person submitting an issue or PR is responsible for its content, +regardless of whether AI tools were used in its creation. Generative AI +tools can produce output quickly, but discretion, good judgment, and +critical thinking are the foundation of all good contributions. We value +good code, concise accurate documentation, and well scoped PRs without +unneeded code churn. + +Considerations for success +========================== + +Authors must review the work done by AI tooling in detail to ensure it +actually makes sense before proposing it as a PR or filing it as an issue. + +We expect PR authors and those filing issues to be able to explain their +proposed changes in their own words. + +Disclosure of the use of AI tools in the PR description is appreciated, +while not required. Be prepared to explain how the tool was used and what +changes it made. + +Whether you are using AI tools or not, keep the following principles in +mind for the quality of your contribution: + +- Consider whether the change is necessary +- Make minimal, focused changes +- Follow existing coding style and patterns +- Write tests that exercise the change +- Keep backwards compatibility with prior releases in mind. Existing + tests may be ensuring specific API behaviors are maintained. + +Pay close attention to AI generated recommendations for testing changes. +Provide input about Python's testing principles when guiding an AI model. +Always review the output before opening a pull request or issue, +including proposed PR or issue titles and descriptions. + +Acceptable uses +=============== + +Some of the acceptable uses of generative AI include: + +- Assistance with writing comments, especially in a non-native language +- Gaining understanding of existing code +- Supplementing contributor knowledge for code, tests, and documentation + +Unacceptable uses +================= + +Maintainers may close issues and PRs that are not useful or productive, +regardless of whether AI tools were used or not. + +If a contributor repeatedly opens unproductive issues or PRs, they may be +blocked from contributing to the project because it is disruptive and +disrespectful of the maintainers time. + +It is not acceptable to alter or bypass existing tests, or remove desired +functionality, in order to make a failing test pass. Such changes are not +a real fix. diff --git a/getting-started/generative-ai.rst b/getting-started/generative-ai.rst deleted file mode 100644 index e4aa3e7586..0000000000 --- a/getting-started/generative-ai.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _generative-ai: - -============= -Generative AI -============= - -Generative AI tools have evolved rapidly, and their suggested results can be helpful. As with using any tool, the resulting contribution is -the responsibility of the contributor. We value good code, concise accurate documentation, and avoiding unneeded code -churn. Discretion, good judgment, and critical thinking are the foundation of all good contributions, regardless of the -tools used in their creation. - -Acceptable uses -=============== - -Some of the acceptable uses of generative AI include: - -- Assistance with writing comments, especially in a non-native language -- Gaining understanding of existing code -- Supplementing contributor knowledge for code, tests, and documentation - -Unacceptable uses -================= - -Maintainers may close issues and PRs that are not useful or productive, including -those that are fully generated by AI. If a contributor repeatedly opens unproductive -issues or PRs, they may be blocked. - -Considerations for success -========================== -- While AI assisted tools such as autocompletion can enhance productivity, they sometimes rewrite entire code blocks instead of making small, focused edits. - This can make it more difficult to review changes and to fully understand both the original intent of the code and the rationale behind the new modifications. - Maintaining consistency with the original code helps preserve clarity, traceability, and meaningful reviews and also helps us avoid unnecessary code churn. -- Sometimes AI assisted tools make failing unit tests pass by altering or bypassing the tests rather than addressing the underlying problem in the code. - Such changes do not represent a real fix. Authors must review the work done by AI tooling in detail to ensure it actually makes sense before proposing it as a PR. -- Keep the following principles for the quality of your contributions in mind whether you use generative AI or not: - - - Consider whether the change is necessary - - Make minimal, focused changes - - Follow existing coding style and patterns - - Write tests that exercise the change diff --git a/getting-started/git-boot-camp.rst b/getting-started/git-boot-camp.rst index b845e00aa6..47f49f3d69 100644 --- a/getting-started/git-boot-camp.rst +++ b/getting-started/git-boot-camp.rst @@ -4,13 +4,7 @@ Git bootcamp and cheat sheet ============================ -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst .. highlight:: console @@ -453,19 +447,33 @@ Both of these tools will configure a remote URL for the branch, so you can ``git push`` if the pull request author checked "Allow edits from maintainers" when creating the pull request. -If you don't have GitHub CLI or hub installed, you can set up a git alias: +Otherwise, you can run the following commands: + +.. code-block:: shell + + $ git fetch upstream pull/NNNNN/head:pr_NNNNN + $ git switch pr_NNNNN + +Or set up a Git alias: .. tab:: Unix/macOS .. code-block:: shell - $ git config --global alias.pr '!sh -c "git fetch upstream pull/${1}/head:pr_${1} && git checkout pr_${1}" -' + git config --global alias.pr '!sh -c "git fetch upstream pull/${1}/head:pr_${1} && git switch pr_${1}" -' -.. tab:: Windows +.. tab:: Windows cmd .. code-block:: dosbatch - git config --global alias.pr "!sh -c 'git fetch upstream pull/${1}/head:pr_${1} && git checkout pr_${1}' -" + git config --global alias.pr "!sh -c 'git fetch upstream pull/${1}/head:pr_${1} && git switch pr_${1}' -" + +.. tab:: Windows Powershell + + .. code-block:: shell + + git config --global alias.pr '!f() { git fetch upstream pull/$1/head:pr_$1 && git checkout pr_$1; }; f' + The alias only needs to be done once. After the alias is set up, you can get a local copy of a pull request as follows:: diff --git a/getting-started/index.rst b/getting-started/index.rst index 05ee67a3bc..48037ad38d 100644 --- a/getting-started/index.rst +++ b/getting-started/index.rst @@ -12,4 +12,4 @@ Getting started git-boot-camp pull-request-lifecycle getting-help - generative-ai + ai-tools diff --git a/getting-started/pull-request-lifecycle.rst b/getting-started/pull-request-lifecycle.rst index d1f7e26a52..78c8cfff06 100644 --- a/getting-started/pull-request-lifecycle.rst +++ b/getting-started/pull-request-lifecycle.rst @@ -27,7 +27,9 @@ Quick guide Here is a quick overview of how you can contribute to CPython: -#. `Create an issue`_ that describes your change [*]_ +#. `Create an issue`_ that describes your change. If it is trivial + (like :ref:`typo fixes <typo-fixes>`), or an issue already exists, + you can skip this step. #. :ref:`Create a new branch in Git <pullrequest-steps>` from the ``main`` branch @@ -51,9 +53,6 @@ Here is a quick overview of how you can contribute to CPython: #. Celebrate contributing to CPython! :) -.. [*] If an issue is trivial (for example, typo fixes), or if an issue already exists, - you can skip this step. - Don't force-push ---------------- @@ -238,8 +237,112 @@ should do to help ensure that your pull request is accepted. changes, because there might be interferences unknown to you between your changes and some other part of the interpreter. +#. **Keep your pull request focused and small.** A pull request should address + one issue or add one feature. Combining multiple unrelated changes makes a + pull request harder to review and increases the number of people notified + unnecessarily. + #. Proper :ref:`documentation <documenting>` additions/changes should be included. +.. _news-entry: +.. _what-s-new-and-news-entries: + +Updating NEWS and What's New in Python +====================================== + +Changes that require NEWS entries +--------------------------------- + +Most changes made to the codebase deserve an entry in :cpy-file:`Misc/NEWS.d`, +except for the following: + +* documentation changes +* test changes +* strictly internal changes with no user-visible effects +* changes that already have a ``NEWS`` entry +* reverts that have not yet been included in any formal release + (including alpha and beta releases) + +For the last two, note the following: + +#. **If a change is reverted prior to release**, then the corresponding + entry is simply removed. Otherwise, a new entry must be added noting + that the change has been reverted (for example, when a feature is released in + an alpha and then cut prior to the first beta). + +#. **If a change is a fix (or other adjustment) to an earlier unreleased + change and the original** ``NEWS`` **entry remains valid**, then no additional + entry is needed. + +Changes that require "What's New in Python" entries +--------------------------------------------------- + +If a change is particularly interesting for end users (for example, new features, +significant improvements, or backwards-incompatible changes), add an entry in +the "What's New in Python" document (in :cpy-file:`Doc/whatsnew/`, the 3.X.rst +file where X is the current Python version) in addition to the ``NEWS`` entry. + +In most cases, it is sufficient to reuse the wording from the ``NEWS`` entry +in the "What's New in Python" entry. + +.. note:: + + A change that needs an entry in "What's New in Python" + is very likely not suitable for inclusion in a maintenance release. + +How to add a NEWS entry +----------------------- + +``NEWS`` entries go into the ``Misc/NEWS.d`` directory as individual files. The +``NEWS`` entry can be created by using `blurb-it <https://blurb-it.herokuapp.com/>`_, +or the :pypi:`blurb` tool and its ``blurb add`` command. + +If you are unable to use the tool, then you can create the ``NEWS`` entry file +manually. The ``Misc/NEWS.d`` directory contains a sub-directory named +``next``, which contains various sub-directories representing classifications +for what was affected (for example, ``Misc/NEWS.d/next/Library`` for changes relating +to the standard library). The file name itself should be in the format +``<datetime>.gh-issue-<issue-number>.<nonce>.rst``: + +* ``<datetime>`` is today's date joined with a hyphen (``-``) to your current + local time, in the ``YYYY-MM-DD-hh-mm-ss`` format (for example, ``2017-05-27-16-46-23``). +* ``<issue-number>`` is the issue number the change is for (for example, ``12345`` + for ``gh-issue-12345``). +* ``<nonce>`` is a unique string to guarantee that the file name is + unique across branches (for example, ``Yl4gI2``). It is typically six characters + long, but it can be any length of letters and numbers. Its uniqueness + can be satisfied by typing random characters on your keyboard. + +As a result, a file name can look something like +``Misc/NEWS.d/next/Library/2017-05-27-16-46-23.gh-issue-12345.Yl4gI2.rst``. + +How to write a NEWS entry +------------------------- + +All ``NEWS`` entries end up being part of the changelog. +The changelog contains *a lot* of entries, +and its intended audience is mainly users, not core devs and contributors. +Take this into consideration when wording your ``NEWS`` entry. +Describe the user-visible effects of your change succinctly and accurately; +avoid long technical elaborations, digressions, and do not expect or require +the reader to have read the actual diff for the change. + +The contents of a ``NEWS`` file should be valid reStructuredText. An 80 character +column width should be used. There is no indentation or leading marker in the +file (for example, ``-``). There is also no need to start the entry with the issue +number since it is part of the file name. You can use +:ref:`inline markups <rest-inline-markup>` too. Here is an example of a ``NEWS`` +entry: + +.. code-block:: rst + + Fix warning message when :func:`os.chdir` fails inside + :func:`test.support.temp_cwd`. Contributed by Chris Jerdonek. + +The inline Sphinx roles like :rst:role:`:func: <py:func>` can be used help readers +find more information. You can build HTML and verify that the +link target is appropriate by using :ref:`make html <building-using-make>`. + Copyrights ========== @@ -260,6 +363,23 @@ See also `python/cpython#126133 <https://github.com/python/cpython/issues/126133#issuecomment-2460824052>`__. +.. _typo-fixes: + +Typo fixes +========== + +Fixing typos and grammatical errors in documentation is a +contribution that does not require a linked issue. The most appropriate +place for such fixes are the :cpy-file:`Doc/` (end-user documentation) +and :cpy-file:`InternalDocs/` directories. + + +Keep typo-fix PRs **small and focused**. Large PRs that touch many unrelated files +notify a large number of reviewers unnecessarily and are harder to review. +Large typo-fix PRs, or PRs touching directories outside those listed above, +may be closed with a reference to this section. + + .. _patchcheck: ``patchcheck`` @@ -285,7 +405,6 @@ The automated checklist runs through: * Has an entry under ``Misc/NEWS.d/next`` been added? (using `blurb-it <https://blurb-it.herokuapp.com/>`__, or the :pypi:`blurb` tool) -* Has ``Misc/ACKS`` been updated? * Has ``configure`` been regenerated, if necessary? * Has ``pyconfig.h.in`` been regenerated, if necessary? @@ -353,8 +472,8 @@ Here are the steps needed in order to sign the CLA: 2. When ``python-cla-bot`` comments on your pull request that commit authors are required to sign a Contributor License Agreement, click - on the button in the comment to sign it. It's enough to log in through - GitHub. The process is automatic. + on the button in the comment to sign it. Log in with GitHub, + click "Authorize Python CLA Bot", and then click "Sign". 3. After signing, the comment by ``python-cla-bot`` will update to indicate that "all commit authors signed the Contributor License diff --git a/getting-started/setup-building.rst b/getting-started/setup-building.rst index df208f28cf..3590e09c3f 100644 --- a/getting-started/setup-building.rst +++ b/getting-started/setup-building.rst @@ -5,13 +5,7 @@ Setup and building ================== -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst .. highlight:: console @@ -89,7 +83,7 @@ You will only need to execute these steps once per machine: (You can use both SSH-based or HTTPS-based URLs.) -.. Step 6 and 7 are are duplicated in bootcamp as well. +.. Step 6 and 7 are duplicated in bootcamp as well. Please update these steps in both places. 6. Add an ``upstream`` remote, then configure ``git`` @@ -197,6 +191,15 @@ Configuration is typically:: More flags are available to ``configure``, but this is the minimum you should do to get a pydebug build of CPython. +.. tip:: + To speed up repeated ``configure`` runs, use ``--config-cache`` (equivalent to ``--cache-file=config.cache``, short: ``-C``):: + + $ ./configure --config-cache --with-pydebug + + This caches results in a :file:`config.cache` file. If you switch compilers or + significantly change your build environment, delete :file:`config.cache` before + re-running ``configure``. + .. note:: You might need to run ``make clean`` before or after re-running ``configure`` in a particular build directory. @@ -385,6 +388,7 @@ compiler just like building for :ref:`Unix <unix-compiling>` as well as: 1. A C compiler that can target WebAssembly (for example, `WASI SDK`_) 2. A WASI host/runtime (for example, Wasmtime_) +3. A system install of Python 3.11 or newer to run the build scripts All of this is provided in the WASI :ref:`dev container <using-a-container>` (which you can select as an alternative container when using a @@ -407,31 +411,43 @@ to have a version of Python for the build system to use and another that's the build you ultimately care about (that is, the build Python is not meant for use by you directly, only the build system). -The easiest way to get a debug build of CPython for WASI is to use the -``Tools/wasm/wasi.py build`` command (which should be run w/ a recent version of -Python you have installed on your machine): +The easiest way to get a debug build of CPython for WASI is to run the +following command with Python 3.11 or newer: -.. tab:: Python 3.14+ +.. tab:: Python 3.15+ .. code-block:: shell - python3 Tools/wasm/wasi build --quiet -- --config-cache --with-pydebug + python Platforms/WASI build --quiet -- --config-cache --with-pydebug + +.. tab:: Python 3.14 + + .. code-block:: shell + + python Tools/wasm/wasi build --quiet -- --config-cache --with-pydebug .. tab:: Python 3.13 .. code-block:: shell - python3 Tools/wasm/wasi.py build --quiet -- --config-cache --with-pydebug + python Tools/wasm/wasi.py build --quiet -- --config-cache --with-pydebug That single command will configure and build both the build Python and the -WASI build in ``cross-build/build`` and ``cross-build/wasm32-wasip1``, -respectively. +WASI build in the ``cross-build/`` directory. You can also do each configuration and build step separately; the command above is a convenience wrapper around the following commands: +.. tab:: Python 3.15+ + + .. code-block:: shell + + $ python Platforms/WASI configure-build-python --quiet -- --config-cache --with-pydebug + $ python Platforms/WASI make-build-python --quiet + $ python Platforms/WASI configure-host --quiet -- --config-cache + $ python Platforms/WASI make-host --quiet -.. tab:: Python 3.14+ +.. tab:: Python 3.14 .. code-block:: shell @@ -454,7 +470,7 @@ is a convenience wrapper around the following commands: The ``configure-host`` command infers the use of ``--with-pydebug`` from the build Python. -Running the separate commands after ``wasi build`` is useful if you, for example, +Running the separate commands after ``build`` is useful if you, for example, only want to run the ``make-host`` step after making code changes. Once everything is complete, there will be a @@ -502,105 +518,110 @@ including browsers and Node.js. CPython being designed for ``./configure`` / ``make``. To build for Emscripten, you will need to cross-compile CPython. This requires a -C compiler just like building for :ref:`Unix <unix-compiling>` as well as: +C compiler just like building for :ref:`Unix <unix-compiling>`. The Node Version +Manager (``nvm``) must also be on your path. -* The Emscripten compiler -* Node.js +Building for Emscripten requires doing a cross-build where you have a *build* +Python to help produce an Emscripten build of CPython. This means you build +CPython twice: once to have a version of Python for the build system to use and +another that's the build you ultimately care about (that is, the build Python is +not meant for use by you directly, only the build system). -The simplest way to install the Emscripten compiler is: +The simplest way to build Emscripten is to run: .. code-block:: sh - # Install Emscripten - git clone https://github.com/emscripten-core/emsdk - ./emsdk/emsdk install 4.0.12 - ./emsdk/emsdk activate 4.0.12 - source ./emsdk/emsdk_env.sh - -Updating the Emscripten compiler version can cause breakages. For the best -compatibility, use the appropriate Emscripten version based on the version of -CPython you're building: - -* For building CPython 3.14, use ``emsdk`` version ``4.0.12``. -* For building the main branch of the CPython repository, you may wish to use - ``latest`` instead of a specific version. + export EMSDK_CACHE=$PWD/cross-build/emsdk + python3 Platforms/emscripten install-emscripten + python3 Platforms/emscripten build all + +``install-emscripten`` downloads and installs the version of the Emscripten SDK +required, placing it in the ``EMSDK_CACHE`` directory. +``build all`` will: + +1. Build a copy of Python that can run on the host machine (the "build" python); +2. Use nvm_ to ensure that the needed version of Node is installed; +3. Download the code for all the binary dependencies of Python (such as + ``libffi`` and ``mpdecimal``), and compile them for Emscripten; and +4. Build a copy of Python that can run on Emscripten (the "host" python). + +The built binary dependencies are cached inside the Emscripten cache directory. +Once built for a given Emscripten version, they will not be rebuilt on +subsequent runs unless there is a change in the version or build script for the +dependency. + +It is assumed that nvm_ is installed in ``${HOME}/.nvm``. If you don't have nvm +installed or don't want to use it, you can pass ``--host-runner node`` to the +``build`` command. The argument should either be the name of an executable that +can be found on the ``PATH`` or a relative or absolute path to an executable. + +If you omit the ``EMSDK_CACHE`` environment variable, the build script will +assume that the current environment has the Emscripten tools available. You are +responsible for downloading and activating those tools in your environment. The +version of Emscripten and Node that is required to build Python is defined in +the :cpy-file:`Platforms/emscripten/config.toml` configuration file. + +There are two environment variables that can be used to control the operation of +the ``Platforms/emscripten`` build script: + +* ``EMSDK_CACHE`` (or the ``--emsdk-cache`` flag) controls the location of the + Emscripten SDK cache directory. You can use this environment variable instead + of passing the ``--emsdk-cache`` flag. When set, the build script will + validate that the required Emscripten version is present in the cache and will + exit with an error if it is not; run ``install-emscripten`` to populate the + cache. +* ``CROSS_BUILD_DIR`` (or the ``--cross-build-dir`` flag) defines the location + of the ``cross-build`` directory that will be used for builds. This can be + useful if you need to maintain builds of multiple versions of Python + side by side. It is possible (but not necessary) to enable ``ccache`` for Emscripten builds -by setting the ``EM_COMPILER_WRAPPER`` environment, but this step will only -take effect if it is done **after** ``emsdk_env.sh`` is sourced (otherwise, the -sourced script removes the environment variable): +by setting the ``EM_COMPILER_WRAPPER`` environment variable: .. code-block:: sh export EM_COMPILER_WRAPPER=ccache -Building for Emscripten requires doing a cross-build where you have a *build* -Python to help produce an Emscripten build of CPython. This means you build -CPython twice: once to have a version of Python for the build system to use and -another that's the build you ultimately care about (that is, the build Python is -not meant for use by you directly, only the build system). - -The easiest way to get a debug build of CPython for Emscripten is to use the -``Tools/wasm/emscripten build`` command, which should be run with a recent -version of Python (3.13 or newer) already installed on your machine: +To get a debug build of CPython for Emscripten, use: .. code-block:: shell - python3 Tools/wasm/emscripten build --quiet -- --config-cache --with-pydebug + python3 Platforms/emscripten build all -- --with-pydebug That single command will configure and build both the build Python and the Emscripten build in ``cross-build/build`` and ``cross-build/wasm32-emscripten/build/python/``, respectively. -You can also do each configuration and build step separately; the command above -is a convenience wrapper around the following commands: - -.. code-block:: shell - - python Tools/wasm/emscripten configure-build-python --quiet -- --config-cache --with-pydebug - python Tools/wasm/emscripten make-build-python --quiet - python Tools/wasm/emscripten make-libffi --quiet - python Tools/wasm/emscripten make-mpdec --quiet - python Tools/wasm/emscripten configure-host --quiet -- --config-cache - python Tools/wasm/emscripten make-host --quiet - -.. note:: - - The ``configure-host`` command infers the use of ``--with-pydebug`` from the - build Python. +The ``Platforms/emscripten`` script has a number of other entry points that allow for +fine-grained execution of each part of an Emscripten build; run +``python3 Platforms/emscripten --help`` for more details. -Running the separate commands after ``emscripten build`` is useful if you, for -example, only want to run the ``make-host`` step after making code changes. -Once everything is complete, there will be a -``cross-build/wasm32-emscripten/build/python/python.sh`` helper file which you -can use to run the ``python.mjs`` file: +Once the build is complete, you can run Python code using: .. code-block:: shell - cross-build/wasm32-emscripten/build/python/python.sh --version + python3 Platforms/emscripten run ./path/to/script.py -You can also use ``Makefile`` targets and they will work as expected thanks to -the ``HOSTRUNNER`` environment variable having been set to a similar value as -used in ``python.sh``: +You can run the CPython test suite using: .. code-block:: shell - make -C cross-build/wasm32-emscripten/build/python/ test + python3 Platforms/emscripten run --test Additional instructions for running the resulting builds (through Node.js and/or through web browsers) are available in the CPython repository at -:cpy-file:`Tools/wasm/README.md`. +:cpy-file:`Platforms/emscripten/README.md`. .. _Emscripten: https://emscripten.org/ .. _WebAssembly: https://webassembly.org - +.. _nvm: https://github.com/nvm-sh/nvm#intro Android ------- Build and test instructions for Android are maintained in the CPython repository -at :cpy-file:`Android/README.md`. +at :cpy-file:`Platforms/Android/README.md`. iOS --- @@ -619,128 +640,108 @@ macOS; then once for each of the three underlying platforms used by iOS: * An ARM64 simulator running on a recent macOS machine; and * An x86_64 simulator running on older macOS machine. -The macOS build is required because building Python involves running some Python -code. On a normal desktop build of Python, you can compile a Python interpreter -and then use that interpreter to run Python code. However, the binaries produced -for iOS won't run on macOS, so you need to provide an external Python -interpreter. From the root of a CPython code checkout, run the following:: - - $ ./configure --prefix=$(pwd)/cross-build/macOS - $ make -j4 all - $ make install - -This will build and install Python for macOS into the ``cross-build/macOS`` -directory. - -The CPython build system can compile a single platform at a time. It is possible -to *test* a single platform at a time; however, for distribution purposes, you -must compile all three, and merge the results. See the `iOS README -<https://github.com/python/cpython/blob/main/iOS/README.rst#merge-thin-frameworks-into-fat-frameworks>`__ -for details on this merging process. - -The following instructions will build CPython for iOS with all extensions -enabled, provided you have installed the build dependencies XZ, BZip2, OpenSSL -and libFFI in subfolders of the ``cross-build`` folder. See :ref:`the iOS -section on installing build dependencies <build-dependencies>` for details on -how to obtain these dependencies. These dependencies are all strictly optional, -however, including libFFI is *highly* recommended, as it is required by the -:py:mod:`ctypes` module which is used on iOS to support accessing native system APIs. - -.. tab:: ARM64 device - - .. code-block:: console - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - LIBLZMA_CFLAGS="-I$(pwd)/cross-build/iphoneos.arm64/xz/include" \ - LIBLZMA_LIBS="-L$(pwd)/cross-build/iphoneos.arm64/xz/lib -llzma" \ - BZIP2_CFLAGS="-I$(pwd)/cross-build/iphoneos.arm64/bzip2/include" \ - BZIP2_LIBS="-L$(pwd)/cross-build/iphoneos.arm64/bzip2/lib -lbz2" \ - LIBFFI_CFLAGS="-I$(pwd)/cross-build/iphoneos.arm64/libffi/include" \ - LIBFFI_LIBS="-L$(pwd)/cross-build/iphoneos.arm64/libffi/lib -lffi" \ - --with-openssl="$(pwd)/cross-build/iphoneos.arm64/openssl" \ - --host=arm64-apple-ios12.0 \ - --build=arm64-apple-darwin \ - --with-build-python=$(pwd)/cross-build/macOS/bin/python3.13 \ - --enable-framework - $ make -j4 all - $ make install - -.. tab:: ARM64 simulator - - .. code-block:: console - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - LIBLZMA_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.arm64/xz/include" \ - LIBLZMA_LIBS="-L$(pwd)/cross-build/iphonesimulator.arm64/xz/lib -llzma" \ - BZIP2_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.arm64/bzip2/include" \ - BZIP2_LIBS="-L$(pwd)/cross-build/iphonesimulator.arm64/bzip2/lib -lbz2" \ - LIBFFI_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.arm64/libffi/include" \ - LIBFFI_LIBS="-L$(pwd)/cross-build/iphonesimulator.arm64/libffi/lib -lffi" \ - --with-openssl="$(pwd)/cross-build/iphonesimulator.arm64/openssl" \ - --host=arm64-apple-ios12.0-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=$(pwd)/cross-build/macOS/bin/python3.13 \ - --enable-framework - $ make -j4 all - $ make install - -.. tab:: x86-64 simulator - - .. code-block:: console - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - LIBLZMA_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.x86_64/xz/include" \ - LIBLZMA_LIBS="-L$(pwd)/cross-build/iphonesimulator.x86_64/xz/lib -llzma" \ - BZIP2_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.x86_64/bzip2/include" \ - BZIP2_LIBS="-L$(pwd)/cross-build/iphonesimulator.x86_64/bzip2/lib -lbz2" \ - LIBFFI_CFLAGS="-I$(pwd)/cross-build/iphonesimulator.x86_64/libffi/include" \ - LIBFFI_LIBS="-L$(pwd)/cross-build/iphonesimulator.x86_64/libffi/lib -lffi" \ - --with-openssl="$(pwd)/cross-build/iphonesimulator.x86_64/openssl" \ - --host=x86_64-apple-ios12.0-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=$(pwd)/cross-build/macOS/bin/python3.13 \ - --enable-framework - $ make -j4 all - $ make install - -These instructions modify your ``PATH`` before the build. As iOS and macOS share -a hardware architecture (ARM64), it is easy for a macOS ARM64 binary to be -accidentally linked into your iOS build. This is especially common when Homebrew -is present on the build system. The most reliable way to avoid this problem is -to remove any potential source of other libraries from your ``PATH``. - -However, the ``PATH`` is not completely bare --- it includes the -``iOS/Resources/bin`` folder. This folder contains a collection of scripts that -wrap the invocation of the Xcode :program:`xcrun` tool, removing user- and -version-specific paths from the values encoded in the :py:mod:`sysconfig` -module. Copies of these scripts are included in the final build products. - -Once this build completes, the ``iOS/Frameworks`` folder will contain a -``Python.framework`` that can be used for testing. - -To run the test suite on iOS, complete a build for a *simulator* platform, -ensure the path modifications from the build are still in effect, and run:: - - $ make testios +You will need an existing Python 3 interpreter to build Python. From the root of +a CPython code checkout, run the following: + +.. tab:: Python 3.15+ + + .. code-block:: shell + + $ python3 Platforms/Apple build iOS all + +.. tab:: Python 3.14 + + .. code-block:: shell + + $ python3 Apple build iOS all + +.. tab:: Python 3.13 + + Python 3.13 requires explicitly invoking ``configure`` and ``make`` for each + platform. For example, to build for the ARM64 simulator, run: + + .. code-block:: shell + + $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + $ ./configure \ + LIBLZMA_CFLAGS="-Ipath/to/xz/include" \ + LIBLZMA_LIBS="-Lpath/to/xz/lib -llzma" \ + BZIP2_CFLAGS="-Ipath/to/bzip2/include" \ + BZIP2_LIBS="-Lpath/to/bzip2/lib -lbz2" \ + LIBFFI_CFLAGS="-Ipath/to/libffi/include" \ + LIBFFI_LIBS="-Lpath/to/libffi/lib -lffi" \ + --with-openssl="path/to/openssl" \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=path/to/python3.13 \ + --enable-framework + $ make -j4 all + $ make install + + The ``--host`` argument should be one of ``arm64-apple-ios-simulator``, + ``x64_64-apple-ios-simulator`` or ``arm64-apple-ios``. Your ``PATH`` should + be kept to a minimum to avoid inadvertently linking ARM64 macOS binaries into + your iOS project. You must specify a path to pre-compiled binary dependencies. + + Once you have built an Apple Framework for each architecture, you will need + to manually construct an XCframework. + +This will: + +1. Build a copy of Python that can run on macOS (the "build" python); +2. Download pre-compiled binaries for dependencies of CPython (such as + ``libFFI`` and ``xz``) +3. Build a copy of Python for each supported iOS architecture (x86_64 simulator, + ARM64 simulator, and ARM64 device); and +4. Generate a release artefact for iOS + +Once this build completes, the ``cross-build/iOS`` folder will contain a +``Python.xcframework``, and the ``cross-build/dist`` folder will contain a +release tarball. + +To run the test suite on iOS, run: + +.. tab:: Python 3.15+ + + .. code-block:: shell + + $ python3 Platforms/Apple test iOS + +.. tab:: Python 3.14 + + .. code-block:: shell + + $ python3 Apple test iOS + +.. tab:: Python 3.13 + + .. code-block:: shell + + $ make testios The full test suite takes approximately 12 minutes to run on a 2022 M1 MacBook Pro, plus a couple of extra minutes to build the testbed application and boot -the simulator. There will be an initial burst of console output while the Xcode -test project is compiled; however, while the test suite is running, there is no -console output or progress. This is a side effect of how Xcode operates when -executed at the command line. You should see an iOS simulator appear during the -testing process; the simulator will booth to an iOS landing screen, the testbed +the simulator. You should see an iOS simulator appear during the +testing process; the simulator will boot to an iOS landing screen, the testbed app will be installed, and then started. The screen of the simulator will be black while the test suite is running. When the test suite completes, success or -failure will be reported at the command line. In the case of failure, you will -see the full log of CPython test suite output. +failure will be reported at the command line. + +Two environment variables can be used to configure the operation of the Apple +build script: + +* ``CACHE_DIR`` defines the location where downloaded artefacts, such + as precompiled ``libFFI`` and ``xz`` binaries, will be stored. +* ``CROSS_BUILD_DIR`` defines the name of the ``cross-build`` directory + that will be used for builds. This can be useful if you need to maintain + builds of multiple versions of Python. + +The ``Platforms/Apple`` script has a number of other entry points that allow for +fine-grained execution of each part of an iOS build; run ``python3 +Platforms/Apple --help`` for more details. You can also run the test suite in Xcode itself. This is required if you want to -run on a physical device; it is also the easiest approach if you need to run a -single test, or a subset of tests. See the `iOS README +run on a physical device. See the `iOS README <https://github.com/python/cpython/blob/main/iOS/README.rst#debugging-test-failures>`__ for details. @@ -856,24 +857,16 @@ some of CPython's modules (for example, ``zlib``). For **Homebrew**, install dependencies using ``brew``:: - $ brew install pkg-config openssl@3 xz gdbm tcl-tk mpdecimal zstd - - .. tab:: Python 3.13+ - - For Python 3.13 and newer:: - - $ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ - GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure --with-pydebug \ - --with-openssl="$(brew --prefix openssl@3)" + $ brew bundle --file=Misc/Brewfile - .. tab:: Python 3.11-3.12 + .. tab:: Python 3.11+ - For Python 3.11 and 3.12:: + For Python 3.11 and newer:: $ GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure --with-pydebug \ + ./configure --config-cache \ + --with-pydebug \ --with-openssl="$(brew --prefix openssl@3)" .. tab:: Python 3.10 @@ -882,7 +875,8 @@ some of CPython's modules (for example, ``zlib``). $ CPPFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ LDFLAGS="-L$(brew --prefix gdbm)/lib -L$(brew --prefix xz)/lib" \ - ./configure --with-pydebug \ + ./configure --config-cache \ + --with-pydebug \ --with-openssl="$(brew --prefix openssl@3)" \ --with-tcltk-libs="$(pkg-config --libs tcl tk)" \ --with-tcltk-includes="$(pkg-config --cflags tcl tk)" \ @@ -904,7 +898,8 @@ some of CPython's modules (for example, ``zlib``). $ GDBM_CFLAGS="-I$(dirname $(dirname $(which port)))/include" \ GDBM_LIBS="-L$(dirname $(dirname $(which port)))/lib -lgdbm" \ - ./configure --with-pydebug \ + ./configure --config-cache \ + --with-pydebug \ --with-system-libmpdec .. tab:: Python 3.11-3.12 @@ -913,7 +908,8 @@ some of CPython's modules (for example, ``zlib``). $ GDBM_CFLAGS="-I$(dirname $(dirname $(which port)))/include" \ GDBM_LIBS="-L$(dirname $(dirname $(which port)))/lib -lgdbm" \ - ./configure --with-pydebug + ./configure --config-cache \ + --with-pydebug And finally, run ``make``:: @@ -942,31 +938,26 @@ some of CPython's modules (for example, ``zlib``). .. tab:: Android The BeeWare project maintains `scripts for building Android dependencies`_, - and distributes `pre-compiled binaries`_ for each of them. + and distributes `pre-compiled Android binaries`_ for each of them. These binaries are automatically downloaded and used by the CPython - build script at :cpy-file:`Android/android.py`. + build script at :cpy-file:`Platforms/Android`. .. _scripts for building Android dependencies: https://github.com/beeware/cpython-android-source-deps - .. _pre-compiled binaries: https://github.com/beeware/cpython-android-source-deps/releases + .. _pre-compiled Android binaries: https://github.com/beeware/cpython-android-source-deps/releases .. tab:: iOS - As with CPython itself, the dependencies for CPython must be compiled for - each of the hardware architectures that iOS supports. Consult the - documentation for `XZ <https://tukaani.org/xz/>`__, `bzip2 - <https://sourceware.org/bzip2/>`__, `OpenSSL <https://www.openssl.org>`__ and - `libffi <https://github.com/libffi/libffi>`__ for details on how to configure - the project for cross-platform iOS builds. - - Alternatively, the `BeeWare Project <https://beeware.org>`__ maintains a - `project for building iOS dependencies - <https://github.com/beeware/cpython-apple-source-deps>`__, and distributes - `pre-compiled binaries - <https://github.com/beeware/cpython-apple-source-deps/releases>`__ for each - of the dependencies. If you use this project to build the dependencies - yourself, the subfolders of the ``install`` folder can be used to configure - CPython. If you use the pre-compiled binaries, you should unpack each tarball - into a separate folder, and use that folder as the configuration target. + The BeeWare project maintains `scripts for building iOS dependencies`_, + and distributes `pre-compiled iOS binaries`_ for each of them. + These binaries are automatically downloaded and used by the CPython + build script at :cpy-file:`Platforms/Apple`. + + If you are building for Python 3.13, you will need to manually download + and install these binaries, and provide the path to the binaries as part + of the call to ``configure``. + + .. _scripts for building iOS dependencies: https://github.com/beeware/cpython-apple-source-deps + .. _pre-compiled iOS binaries: https://github.com/beeware/cpython-apple-source-deps/releases .. c_install_dependencies_end diff --git a/include/activate-tab.rst b/include/activate-tab.rst new file mode 100644 index 0000000000..229542bf42 --- /dev/null +++ b/include/activate-tab.rst @@ -0,0 +1,7 @@ +.. raw:: html + + <script> + document.addEventListener('DOMContentLoaded', function() { + activateTab(getOS()); + }); + </script> diff --git a/index.rst b/index.rst index 997f599327..f6bd875f06 100644 --- a/index.rst +++ b/index.rst @@ -1,30 +1,85 @@ .. _devguide-main: +.. _contributing: ======================== Python Developer's Guide ======================== -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: include/activate-tab.rst .. highlight:: bash -This guide is a comprehensive resource for :ref:`contributing <contributing>` +This guide is a comprehensive resource for contributing to Python_ -- for both new and experienced contributors. It is :ref:`maintained <devguide>` by the same -community that maintains Python. We welcome your contributions to Python! +community that maintains Python. We welcome your contributions! + +Start with the area that best matches what you want to do. +If you still have questions after reviewing the material in +this guide, then the `Core Python Mentorship`_ group is available to help guide new +contributors through the process. + +.. list-table:: + :widths: 10 10 10 + :header-rows: 1 + + * - Documentation + - Code + - Triage + * - + * :ref:`docquality` + * :ref:`documenting` + * :ref:`style-guide` + * :ref:`rst-primer` + * :ref:`translating` + * :ref:`devguide` + * :ref:`ai-tools` + - + * :ref:`setup` + * :ref:`help` + * :ref:`pullrequest` + * :ref:`runtests` + * :ref:`fixingissues` + * :ref:`communication` + * :ref:`gitbootcamp` + * :ref:`devcycle` + * :ref:`ai-tools` + - + * :ref:`tracker` + * :ref:`triaging` + * :ref:`helptriage` + * :ref:`experts` + * :ref:`labels` + * :ref:`gh-faq` + * :ref:`triage-team` + +We **recommend** that sections of this guide be read as needed. You +can stop where you feel comfortable and begin contributing immediately without +reading and understanding everything. If you do choose to skip +around within the guide, be aware that some sections build on each other, +so you may need to backtrack for missing concepts or terminology. + +For broader open-source contribution advice, a number of individuals from the +Python community have contributed to a series +of excellent guides at `Open Source Guides <https://opensource.guide/>`__. +For example, +`How to Contribute to Open Source <https://opensource.guide/how-to-contribute/>`__. + +.. admonition:: You don't have to be a compiler engineer to work on Python -- Savannah Ostrowski + + .. container:: compact-video + + .. youtube:: WGXXxGLBVF4 + :privacy_mode: + :width: 100% + .. _quick-reference: Quick reference --------------- -Here are the basic steps needed to get set up and contribute a pull request. +Here are the basic steps needed to get set up and open a pull request. This is meant as a checklist, once you know the basics. For complete instructions please see the :ref:`setup guide <setup>`. @@ -43,13 +98,13 @@ instructions please see the :ref:`setup guide <setup>`. .. code-block:: shell - ./configure --with-pydebug && make -j $(nproc) + ./configure --config-cache --with-pydebug && make -j $(nproc) .. tab:: macOS .. code-block:: shell - ./configure --with-pydebug && make -j8 + ./configure --config-cache --with-pydebug && make -j8 .. tab:: Windows @@ -76,9 +131,10 @@ instructions please see the :ref:`setup guide <setup>`. ./python.exe -m test -j8 - Note: :ref:`Most <mac-python.exe>` macOS systems use - :file:`./python.exe` in order to avoid filename conflicts with - the ``Python`` directory. + .. note:: + :ref:`Most <mac-python.exe>` macOS systems use + :file:`./python.exe` in order to avoid filename conflicts with + the ``Python`` directory. .. tab:: Windows @@ -91,40 +147,18 @@ instructions please see the :ref:`setup guide <setup>`. git checkout -b fix-issue-12345 main If an issue does not already exist, please `create it - <https://github.com/python/cpython/issues>`__. Trivial issues (for example, typo fixes) do - not require any issue to be created. - -6. Once you fixed the issue, run the tests, and the patchcheck: - - .. tab:: Unix - - .. code-block:: shell - - make patchcheck - - .. tab:: macOS - - .. code-block:: shell - - make patchcheck - - .. tab:: Windows - - .. code-block:: dosbatch - - .\python.bat Tools\patchcheck\patchcheck.py + <https://github.com/python/cpython/issues>`__. Trivial issues (for example, typos) do + not require an issue. - If everything is ok, commit. - -7. Push the branch on your fork on GitHub and :ref:`create a pull request - <pullrequest>`. Include the issue number using ``gh-NNNN`` in the - pull request description. For example: +6. Push the branch on your fork on GitHub and :ref:`create a pull request + <pullrequest>`. Include the issue number using ``gh-NNNNNN`` in the + pull request title. For example: .. code-block:: text gh-12345: Fix some bug in spam module -8. Add a News entry into the ``Misc/NEWS.d`` directory as individual file. The +7. Add a News entry into the ``Misc/NEWS.d/`` directory as individual file. The news entry can be created by using `blurb-it <https://blurb-it.herokuapp.com/>`__, or the :pypi:`blurb` tool and its ``blurb add`` command. Please read more about ``blurb`` in its @@ -136,58 +170,6 @@ instructions please see the :ref:`setup guide <setup>`. Agreement (CLA) as described in the :ref:`Licensing <cla>` section of this guide. -Quick links ------------ - -Here are some links that you probably will reference frequently while -contributing to Python: - -* `Issue tracker`_ -* `Buildbot status`_ -* :ref:`help` -* PEPs_ (Python Enhancement Proposals) -* :ref:`gitbootcamp` - -.. _contributing: - -Contributing ------------- - -We encourage everyone to contribute to Python and that's why we have put up this -developer's guide. If you still have questions after reviewing the material in -this guide, then the `Core Python Mentorship`_ group is available to help guide new -contributors through the process. - -A number of individuals from the Python community have contributed to a series -of excellent guides at `Open Source Guides <https://opensource.guide/>`__. - -Core developers and contributors alike will find the following guides useful: - -* `How to Contribute to Open Source <https://opensource.guide/how-to-contribute/>`__ -* `Building Welcoming Communities <https://opensource.guide/building-community/>`__ - -Guide for contributing to Python: - -======================== =================== ======================= ======================= -Contributors Documentarians Triagers Core team -======================== =================== ======================= ======================= -:ref:`setup` :ref:`docquality` :ref:`tracker` :ref:`responsibilities` -:ref:`help` :ref:`documenting` :ref:`triaging` :ref:`developers` -:ref:`pullrequest` :ref:`style-guide` :ref:`helptriage` :ref:`committing` -:ref:`runtests` :ref:`rst-primer` :ref:`experts` :ref:`devcycle` -:ref:`fixingissues` :ref:`translating` :ref:`labels` :ref:`motivations` -:ref:`communication` :ref:`devguide` :ref:`gh-faq` :ref:`experts` -:ref:`gitbootcamp` :ref:`triage-team` -:ref:`devcycle` -======================== =================== ======================= ======================= - -We **recommend** that the documents in this guide be read as needed. You -can stop where you feel comfortable and begin contributing immediately without -reading and understanding these documents all at once. If you do choose to skip -around within the documentation, be aware that it is written assuming preceding -documentation has been read so you may find it necessary to backtrack to fill in -missing concepts and terminology. - Proposing changes to Python itself ---------------------------------- @@ -206,36 +188,6 @@ happen and that process is also described as part of this guide: * :ref:`langchanges` -Other interpreter implementations ---------------------------------- - -This guide is specifically for contributing to the Python reference interpreter, -also known as CPython (while most of the standard library is written in Python, -the interpreter core is written in C and integrates most easily with the C and -C++ ecosystems). - -There are other Python implementations, each with a different focus. Like -CPython, they always have more things they would like to do than they have -developers to work on them. Some major examples that may be of interest are: - -* PyPy_: A Python interpreter focused on high speed (JIT-compiled) operation - on major platforms. -* GraalPy_: A Python interpreter which has first-class support for - embedding in Java, built on GraalVM. -* Jython_: A Python interpreter focused on good integration with the Java - Virtual Machine (JVM) environment. -* IronPython_: A Python interpreter focused on good integration with the - Common Language Runtime (CLR) provided by .NET and Mono. -* Stackless_: A Python interpreter focused on providing lightweight - microthreads while remaining largely compatible with CPython specific - extension modules. -* MicroPython_: A tiny Python interpreter with small subset of the Python - standard library that is optimised to run on microcontrollers and in - constrained environments. -* CircuitPython_: A fork of MicroPython designed to simplify experimenting - and learning to code on low-cost microcontroller boards. - - Key resources ------------- @@ -252,11 +204,14 @@ Key resources * Source code * `Browse online <https://github.com/python/cpython/>`__ - * `Snapshot of the *main* branch <https://github.com/python/cpython/archive/main.zip>`__ + * `Download a snapshot <https://github.com/python/cpython/archive/main.zip>`__ + of the ``main`` branch * PEPs_ (Python Enhancement Proposals) +* `The Python Discourse <https://discuss.python.org/>`__ * :ref:`help` * :ref:`developers` +* :ref:`gitbootcamp` .. _resources: @@ -274,30 +229,52 @@ Additional resources * :ref:`gdb` * :ref:`clang` - * Various tools with configuration files as found in the `Misc directory`_ - * Information about editors and their configurations can be found in the - `wiki <https://wiki.python.org/moin/PythonEditors>`__ + * Various tools with configuration files as found in the + :cpy-file:`Misc/` directory * `python.org maintenance`_ -* :ref:`Search this guide <search>` Code of conduct --------------- + Please note that all interactions on `Python Software Foundation <https://www.python.org/psf-landing/>`__-supported infrastructure is `covered <https://www.python.org/psf/records/board/minutes/2014-01-06/#management-of-the-psfs-web-properties>`__ by the `PSF Code of Conduct <https://policies.python.org/python.org/code-of-conduct/>`__, which includes all infrastructure used in the development of Python itself -(for example, mailing lists, issue trackers, GitHub, etc.). +(for example, Discourse, issue trackers, GitHub, and so on). In general this means everyone is expected to be open, considerate, and respectful of others no matter what their position is within the project. -Status of Python branches -------------------------- -Moved to :ref:`versions` +Other interpreter implementations +--------------------------------- + +This guide is specifically for contributing to the Python reference interpreter, +also known as CPython (while most of the standard library is written in Python, +the interpreter core is written in C and integrates most easily with the C and +C++ ecosystems). + +There are other Python implementations, each with a different focus. Like +CPython, they always have more things they would like to do than they have +developers to work on them. Some major examples that may be of interest are: + +* PyPy_: A Python interpreter focused on high speed (JIT-compiled) operation + on major platforms. +* GraalPy_: A Python interpreter which has first-class support for + embedding in Java, built on GraalVM. +* Jython_: A Python interpreter focused on good integration with the Java + Virtual Machine (JVM) environment. +* IronPython_: A Python interpreter focused on good integration with the + Common Language Runtime (CLR) provided by .NET and Mono. +* MicroPython_: A tiny Python interpreter with small subset of the Python + standard library that is optimised to run on microcontrollers and in + constrained environments. +* CircuitPython_: A fork of MicroPython designed to simplify experimenting + and learning to code on low-cost microcontroller boards. + .. _contents: @@ -314,12 +291,12 @@ Full table of contents testing/index development-tools/index core-team/index + security/index internals versions - contrib/index -.. _Buildbot status: https://www.python.org/dev/buildbot/ -.. _Misc directory: https://github.com/python/cpython/tree/main/Misc + +.. _Buildbot status: https://buildbot.python.org/ .. _PEPs: https://peps.python.org/ .. _python.org maintenance: https://pythondotorg.readthedocs.io/ .. _Python: https://www.python.org/ @@ -328,7 +305,6 @@ Full table of contents .. _GraalPy: https://www.graalvm.org/python/ .. _Jython: https://www.jython.org/ .. _IronPython: https://ironpython.net/ -.. _Stackless: https://github.com/stackless-dev/stackless/wiki/ .. _MicroPython: https://micropython.org/ .. _CircuitPython: https://circuitpython.org/ .. _Issue tracker: https://github.com/python/cpython/issues diff --git a/requirements.txt b/requirements.txt index 7c2bbd806e..789f9597a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ -furo>=2022.6.4 +furo>=2025.12.19 jinja2 -sphinx-autobuild>=2024.9.19 -sphinx-inline-tabs>=2023.4.21 +linklint +sphinx-autobuild>=2025.8.25 +sphinx-inline-tabs>=2025.12.21.14 +sphinx-last-updated-by-git>=0.3.8 sphinx-lint==1.0.2 -sphinx-notfound-page>=1.0.0 -sphinx_copybutton>=0.3.3 +sphinx-notfound-page>=1.1.0 +sphinx_copybutton>=0.5.2 +sphinxcontrib-youtube>=1.5.0 sphinxext-opengraph>=0.13.0 sphinxext-rediraffe -Sphinx>=8.2.3 +Sphinx>=9.1.0 diff --git a/security/index.rst b/security/index.rst new file mode 100644 index 0000000000..dd084b9eb0 --- /dev/null +++ b/security/index.rst @@ -0,0 +1,12 @@ +.. _security: + +======== +Security +======== + +.. toctree:: + :maxdepth: 5 + + policy + psrt + sbom diff --git a/security/policy.rst b/security/policy.rst new file mode 100644 index 0000000000..8dc736d24c --- /dev/null +++ b/security/policy.rst @@ -0,0 +1,167 @@ +=============== +Security policy +=============== + +.. important:: + + :ref:`Python Security Response Team <psrt>` (PSRT) members balance this work + against many other responsibilities. Please be thoughtful about the time and + attention your report requires. Repeated failure to respect the security policy + will result in future reports being rejected, or the reporter being banned + from the ``python`` GitHub organization, regardless of technical merit. + +What types of bugs are vulnerabilities? +--------------------------------------- + +**Not all bugs are vulnerabilities.** + +To avoid causing duplicate work for PSRT members, **all potential** reports +must be evaluated against the relevant threat models +prior to being submitted to the PSRT. +Where possible, cite the relevant threat model to show that +the latter has been considered while determining whether +to report a bug as a vulnerability. + +Vulnerabilities must be exploitable from code, configurations, +pre-conditions, or deployments that may exist in the real world. +A vulnerability that only affects code unlikely to be used +in a production program will not be accepted. + +Documented functionality is not considered a vulnerability. +For example, :mod:`pickle`, :mod:`marshal`, :mod:`shelve`, :func:`eval`, +and :func:`exec` are documented to execute arbitrary Python code that is +supplied as data. The :mod:`ctypes` module is documented to enable modifying +arbitrary locations in memory. + +Vulnerabilities must not depend on malicious control of Python's launch +conditions, including (but not limited to) command line arguments, environment variables, or +modifications to files on the target system. We assume that, at the time Python +is executed, the environment is as intended by the legitimate user, and any +malicious variation from this cannot be mitigated by Python itself. + +Vulnerabilities that affect availability (such as DoS, ReDoS, crashes, +dead-locks, and resource exhaustion) must be +triggerable with data inputs that are reasonably sized for the use case. +Availability vulnerabilities must also demonstrate an "upward" change in posture +for the attacker, rather than a "lateral" one. +This is to avoid handling performance improvements as security vulnerabilities. + +Vulnerabilities in dependencies of Python (such as zlib, Tcl/Tk, or OpenSSL) +are not vulnerabilities in Python unless Python's use of the dependency +interferes with secure use of the dependency. +For example, a vulnerability in the bundled copy of zlib in Python is a +vulnerability in zlib, not Python. + +What versions of Python accept reports? +--------------------------------------- + +Python accepts vulnerability reports and will +assign CVE IDs for :ref:`supported Python versions <branchstatus>` that have a +status of :ref:`"bugfix" or "security" <version-status-key>`. Versions that are +not yet stable (status of :ref:`"feature" or "prerelease" <version-status-key>`) +are not eligible for CVE IDs. If the vulnerability only exists in prerelease +versions (alphas, betas, release candidates), then the issue should be reported +as a regular bug. +Prior to submitting a report, check whether the issue has already been +resolved on the ``main`` branch and only requires backporting. + +Sometimes features may be marked as +"experimental" in Python, even in a stable Python version. +These features are not eligible for security vulnerabilities. +Instead, open a public GitHub issue. + +If a vulnerability is platform-dependent, check if the platform is +supported per :pep:`11`. +Vulnerabilities that exclusively affect unsupported platforms +are not treated as vulnerabilities in Python. + +As per the :pep:`Unsupported Platforms section of PEP 11 <11#unsupported-platforms>`, +porting Python to an unsupported platform is treated as a third-party project. +If you choose to report such a vulnerability to Python, please follow the +requirements of this guide. Note that these reports may be shared with +parties who expressed interested in the relevant platforms and will +generally be handled according to the relevant maintainers' security +policies. These reports may closed if the maintainers are unknown or +unresponsive. + +What to include and how to structure a vulnerability report? +------------------------------------------------------------ + +For your vulnerability report to be handled efficiently by +the PSRT, the report must include certain information and +be formatted correctly: + +* For the initial report and follow-up communications, avoid + overly long, verbose, or excessive structure (such as headers or tables). + Reports should be a few sentences describing the vulnerability. Ideally include + a proof-of-concept script that reproduces the issue and provides a clear + indication of whether the vulnerability is still present (such as exiting with + ``1`` if vulnerable and ``0`` if not vulnerable). +* When reporting large numbers or "batches" of vulnerabilities or + searching for potential vulnerabilities using an LLM, you as a reporter must + verify the factual validity (such as whether APIs have been hallucinated) + of the content in all reports prior to submission to the PSRT. +* Do not include severity or CVSS information in your initial report, + this information will be determined by the PSRT. +* Ideally, include a minimal patch with the mitigation for the report. +* Always include the versions of Python that were tested, + and indicate which were found to be vulnerable. +* Submit reports as plain-text only, including attachments. + No PDFs, binaries, notebooks, or other files that cannot be safely reviewed. + If your proof-of-concept depends on a specially constructed binary file, + please include a script to construct it rather than the file itself. +* Proof-of-concept scripts longer than a few lines should be wrapped with a + `collapsed section`_ using ``<details></details>`` for better readability. +* Reports that do not contain a potential security vulnerability (such as spam + or requesting compliance or due-diligence work) + will be discarded without a reply. + +.. _collapsed section: https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections + +How to submit a vulnerability report? +------------------------------------- + +Submit all potential security vulnerability reports for CPython +by `opening a new ticket <GHSA_>`_. +Do not open a public GitHub issue to report a security vulnerability. +For all other projects (such as pip, python.org and tools) or if you're +not sure where to send your report, send an email to +`security@python.org <mailto:security@python.org>`__. + +Here's what to expect for how a vulnerability report will be handled: + +* Reporter reports the vulnerability privately to the PSRT. +* If the PSRT determines the report isn't a vulnerability, the reporter + may open a public issue. +* If the PSRT determines the report is a vulnerability, the PSRT will + accept the report and a CVE ID will be assigned by the PSF CNA. +* Once a public pull request containing a fix is merged to CPython, + the advisory and CVE record will be published with attribution. + +For more information about how the PSRT handles vulnerabilities, +see :ref:`psrt-vulnerability-process`. + +Code of conduct +--------------- + +Well-being and safety of the Python Security Response Team members is +prioritized over the technical merit of vulnerability reports. +Despite communications being private, vulnerability reporting is subject +to the `PSF Code of Conduct`_. Violations will be reported to the Code of +Conduct team with undisclosed vulnerability information removed, if applicable. + +.. _GHSA: https://github.com/python/cpython/security/advisories/new +.. _PSF Code of Conduct: https://policies.python.org/python.org/code-of-conduct/ + +CVE Numbering Authority (CNA) +----------------------------- + +The Python and pip projects are scoped under the +`Python Software Foundation CVE Numbering Authority <CNA>`__ +(CNA). This means you must submit all security +vulnerability reports to the PSRT for a CVE ID +to be issued for Python or pip. To reach the PSF +CNA contact directly, send an email to +`cna@python.org <mailto:cna@python.org>`__. + +.. _CNA: https://www.python.org/cve-numbering-authority/ diff --git a/security/psrt-emeritus.csv b/security/psrt-emeritus.csv new file mode 100644 index 0000000000..f9d26bfba4 --- /dev/null +++ b/security/psrt-emeritus.csv @@ -0,0 +1,10 @@ +Anthony Baxter,,Release Manager +Christian Heimes,tiran, +Georg Brandl,birkenfeld, +Huzaifa Sidhpurwala,, +Jesse Noller,, +Kushal Das,kushaldas, +Mark Hammond,mhammond, +Nam Nguyen,, +Neal Norwitz,, +Trent Mick,, diff --git a/security/psrt.csv b/security/psrt.csv new file mode 100644 index 0000000000..46391d296c --- /dev/null +++ b/security/psrt.csv @@ -0,0 +1,30 @@ +Adam Turner,AA-Turner, +Barry Warsaw,warsaw,Admin +Bénédikt Tran,picnixz, +Benjamin Peterson,benjaminp, +Damian Shaw,notatallshaw, +Donald Stufft,dstufft, +Dustin Ingram,di, +Ee Durbin,ewdurbin,Admin +Emma Smith,emmatyping, +Glyph Lefkowitz,glyph, +Gregory P. Smith,gpshead, +Hugo van Kemenade,hugovk,Release Manager +Jacob Coffee,JacobCoffee, +Jelle Zijlstra,JelleZijlstra, +Kirill Podoprigora,eclips4, +Larry Hastings,larryhastings, +Łukasz Langa,ambv,Release Manager +Ned Deily,ned-deily,"Admin, Release Manager" +Pablo Galindo Salgado,pablogsal,Release Manager +Paul McMillan,paulmcmillan, +Petr Viktorin,encukou, +Pradyun Gedam,pradyunsg, +Savannah Ostrowski,savannahostrowski,Release Manager +Serhiy Storchaka,serhiy-storchaka, +Seth Larson,sethmlarson,Admin +Stan Ulbrych,StanFromIreland, +Steve Dower,zooba,Release Manager +Thomas Wouters,Yhg1s,Release Manager +Tim Peters,tim-one, +William Woodruff,woodruffw, diff --git a/security/psrt.rst b/security/psrt.rst new file mode 100644 index 0000000000..6a869de6ad --- /dev/null +++ b/security/psrt.rst @@ -0,0 +1,379 @@ +.. _psrt: + +Python Security Response Team (PSRT) +==================================== + +The Python Security Response Team (PSRT) is responsible for handling +vulnerability reports for CPython and pip. + +To report security issues: https://www.python.org/dev/security/ + +Members +------- + +The PSRT publishes a full +list of members and admins, included in the table below: + +.. csv-table:: + :header: "Name", "GitHub username", "Notes" + :file: psrt.csv + :encoding: "utf-8" + +See also the :ref:`members emeritus list <psrt-members-emeritus>`. + +How can I join the PSRT? +~~~~~~~~~~~~~~~~~~~~~~~~ + +Anyone can join the PSRT following a nomination process +`similar to core team nominations`_. Nomination for a new member +is brought to the PSRT by an existing PSRT member and then +this nomination is voted on by existing PSRT members. +The nomination succeeds if the nomination receives at least +two-thirds positive votes from a vote of existing PSRT members +that is open for one week and not vetoed by the Steering Council. + +Once per year the Steering Council will receive a report of inactive members +of the PSRT with the recommendation to remove the inactive users from the PSRT. +“Inactive” is defined as a member who hasn’t coordinated or commented on a +vulnerability report in the past year since the last report was generated. +The Steering Council may remove members of the PSRT with a simple vote. + +Members of the PSRT who are a Release Manager or Steering Council member may +remain in the PSRT regardless of inactivity in vulnerability reports. + +.. _similar to core team nominations: https://devguide.python.org/core-team/join-team/ + +Responsibilities of PSRT members +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Below are the responsibilities of PSRT members: + +* Being knowledgeable about typical software vulnerability report handling + processes, such as CVE IDs, patches, coordinated disclosure, embargoes. +* Not sharing or acting on embargoed information about the reported + vulnerability. Examples of disallowed behavior include sharing information + with colleagues or publicly deploying unpublished mitigations or patches ahead + of the advisory publication date. +* Acting as a “Coordinator” of vulnerability reports that are submitted to + projects. A Coordinator’s responsibility is to move a report through the PSRT + process to a “finished” state, either rejected or as a published advisory and + mitigation, within the industry standard timeline of 90 days. +* As a Coordinator, involving relevant core team members or triagers where + necessary to make a determination whether a report is a vulnerability and + developing a patch. Coordinators are encouraged to involve members of the core + team to make the best decision for each report rather than working in isolation. +* As a Coordinator, calculating the severity using CVSSv4 and authoring advisories + to be shared on `security-announce@python.org`_. These advisories are used for + CVE records by the `PSF CVE Numbering Authority`_. +* Coordinators that can no longer move a report forwards for any reason must + delegate their Coordinator role to someone else in the PSRT. + +.. _security-announce@python.org: https://mail.python.org/archives/list/security-announce@python.org/ +.. _PSF CVE Numbering Authority: https://www.python.org/cve-numbering-authority/ + +Responsibilities of PSRT admins +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +PSRT members who are designated as admins by the Steering Council have the +following additional responsibilities: + +* Triaging the ``security@python.org`` mailing list. +* Managing PSRT membership access including the GitHub team, the mailing list, + and Discord channel, to ensure they are synchronized with the canonical list + of PSRT members. +* On a yearly basis, providing the Steering Council with a report including a + list of inactive PSRT members. +* Running nomination elections, including counting final votes and giving + the Steering Council an opportunity to veto nominations via email. + +.. _psrt-vulnerability-process: + +Triaging a vulnerability report +------------------------------- + +PSRT members coordinate reports from when they are first submitted +to a "finished" state. Finished states include +marking a report as a "non-issue", opening a public issue on GitHub, +or a merged patch with an accompanying CVE and advisory to +``security-announce@python.org``. Reports should reach a finished +state within 90 days of being received by the PSRT. + +Reports enter the system through ``security@python.org`` or +on a project GitHub Security Advisory (GHSA) ticketing system. +For projects that use GHSA, reports to ``security@python.org`` +should have reporters :ref:`re-open their report using GHSA <submit-using-ghsa>`. + +.. raw:: html + :file: ../_static/psrt-coordinator-report.html + +New report in GitHub Security Advisories (GHSA) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Once a report is in GHSA, a "Coordinator" must be assigned +to be responsible for moving the report through the process. +The "Coordinator" role is assigned using a "Credit" in a GHSA ticket +(Select 'Edit' > 'Credit' > Add GitHub username and the role 'Coordinator'). + +.. warning:: + Assigning the "Coordinator" role to each GHSA ticket is important, + as this metadata records whether a PSRT member is + `"active" according to PEP 811`_ to avoid being removed due to inactivity. + +.. _"active" according to PEP 811: https://peps.python.org/pep-0811/#psrt-membership-policy + +If a GHSA ticket is idle for three days without a coordinator +assigned a PSRT member who is not a Release Manager +or Steering Council member will be automatically assigned +as coordinator by the PSRT bot. +If a coordinator can't complete the process +they must find a replacement coordinator in the PSRT +and re-assign the GHSA ticket. + +.. raw:: html + :file: ../_static/psrt-coordinator-triage.html + +Determining whether a report is a vulnerability +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The coordinator will make a determination about a report, either +marking the ticket as a non-issue, as an issue that isn't a security +vulnerability, or as a security vulnerability. If the Coordinator needs +help from core team experts in making the determination, the +experts may be added as 'Collaborators' to the GHSA ticket. +Accepted security vulnerabilities will be moved to the 'Draft' state in GHSA. + +If the report isn't a vulnerability, coordinators close the GHSA ticket +after optionally opening a public GitHub issue. Note that reporters often +will not open a GitHub issue on their own, as there is no longer an incentive +for them to do so without a CVE being assigned. + +Remediating a vulnerability report +---------------------------------- + +Once a report has been accepted as a vulnerability, the remediation +development process begins. Coordinators move the GHSA ticket to a 'Draft' +state using the green 'Accept as Draft' button. Once in this state, +the PSRT bot will automatically assign a CVE ID from the Python Software +Foundation CVE Numbering Authority. + +Once a vulnerability has been accepted there are three things +the Coordinator must prepare before sending an advisory and +closing the GHSA ticket: + +* Severity calculated using CVSSv4. +* Pull request containing the fix merged with a public GitHub issue. +* Advisory title and short description of the vulnerability. + +Severity scoring +~~~~~~~~~~~~~~~~ + +Severity of a vulnerability can be difficult to assess +objectively due to not knowing how software is used +in all situations. Severity is calculated from expected +or known use, not from worst-case hypothetical scenarios. + +The PSRT and PSF CNA use `CVSSv4`_ for calculating +the severity of a vulnerability. GHSA tickets provide a +CVSSv4 calculator within the ticket UI. Note that GitHub defaults to CVSSv3, +change the scoring algorithm to CVSSv4 before scoring in the ticket UI. +As with all aspects of PSRT operations, coordinators are encouraged to ask +for help in calculating a severity from other PSRT members. + +.. _CVSSv4: https://www.first.org/cvss/v4.0/ + +Developing a patch privately +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patch development can initially be done privately by selecting the +'Start a temporary private fork' button within the GHSA ticket. Note that +due to the size of Git repositories, this fork repository may +take several minutes to create. Once the fork has been created any PSRT member +or GHSA collaborator can clone the fork and develop a fix and push a branch: + +.. code-block:: shell + + git clone https://github.com/python/cpython-ghsa-abcd-efgh-ijkl.git + cd cpython-ghsa-abcd-efgh-ijkl + git checkout -b advisory-fix-1 + + # (develop a fix) + + git add ... + git commit -m "Fix" + git push origin advisory-fix-1 + +From here a pull request can be opened within the GHSA ticket +to be reviewed privately. The coordinator can add core team experts +as collaborators as necessary to the GHSA to develop the patch and to review +the patch if the expertise for patch development or review +aren't available within PSRT. Coordinators are not necessarily responsible +for developing patches themselves: only with seeing that a patch is +developed in a timely fashion by involving experts as necessary. + +Once the patch has been approved +a public GitHub issue and pull request can be opened. +Generate a blurb in the category ``Security`` for the public pull request. +Public issues and pull requests for security fixes should use the +:gh-label:`type-security` label. + +To quickly pull a patch file from a GHSA pull request, you +can append ``.patch`` to the pull request URL, like so: + +.. code-block:: shell + + curl https://github.com/python/cpython-ghsa-abcd-efgh-ijkl/pull/1.patch \ + --output ghsa-abcd-efgh-ijkl.patch + +This patch can then be applied and pushed to the public GitHub repository: + +.. code-block:: shell + + git remote -v + origin ssh://git@github.com/.../cpython (fetch) + origin ssh://git@github.com/.../cpython (fetch) + + git checkout -b branch-name + git apply ./ghsa-abcd-efgh-ijkl.patch + git push origin branch-name + +.. important:: CPython's backport infrastructure + is used for tracking backported patches. Use **one GitHub issue + per CVE** to accurately track backports of vulnerability fixes. + For new CVEs, even when related to a previous issue, **open a + new GitHub issue** to accurately track fixed versions. + +.. important:: Don't select the green 'Merge pull request' + or 'Publish advisory' buttons within GHSA. Advisories are published + to the mailing list, and the 'Merge pull request' button within + GHSA bypasses all continuous integration and branch protection + steps. Use a public pull request instead. + +Publishing an advisory +~~~~~~~~~~~~~~~~~~~~~~ + +Once the vulnerability fix has been merged into the main branch via a +public GitHub pull request, an advisory must be published. +The advisory requires the severity, +a title, and a short description of the vulnerable module, functions, +behavior and fix. This short description can optionally include mitigation steps +if applying the patch isn't the only way to mitigate the vulnerability. + +* Send an email to the ``security-announce@python.org`` mailing list + using the `advisory template`_, including title, severity, description. +* The advisory email will be received by PSF CVE Numbering Authority + operators and used to publish a CVE record. +* Begin the backporting process for all Python branches still receiving + security updates. Add the :gh-label:`type-security` and :gh-label:`release-blocker` labels + to each backport pull request so that release managers can find them prior + to releasing. + +After an advisory email is sent, the GHSA ticket can be closed. + +.. _advisory template: #advisory-email + +Handling code signing certificate reports +----------------------------------------- + +Python signs binaries using Azure Trusted Signing and Apple Developer ID +certificates. If a code signing certificate is reported as "compromised" or +"malware signed with certificate", the Python Security Response Team must +request the following information from the reporter: + +* Checksum(s) of binaries signed by certificate. +* Signature(s) of binaries signed by certificate. + +To avoid unnecessary user confusion and churn around revoking code signing +certificates, any reports **must be verifiable independently by the PSRT before +taking destructive actions**, such as revoking certificates. With this +information the PSRT can take investigative steps to verify the report, such as: + +* Downloading and checking artifacts from the associated Azure Pipelines + executions against the reported list of checksums. +* Verifying the validity of the signatures. `Past reports + <https://discuss.python.org/t/103356/2>`__ have contained signatures that + purported to be from Python code signing certificates, but were not valid. +* Checking the Azure Pipelines and Azure Trusted Signing audit logs for signs of + compromise. + +If any signs of compromise or incorrectly signed binaries are discovered by the +PSRT, only then will certificates be revoked and an advisory published. +If compromise is reported, the following non-destructive actions can be taken by +the PSRT without verifying the reported information as a precaution, if +relevant: + +* Rotating secrets associated with code signing (``TrustedSigningSecret`` for + Azure Trusted Publishing). +* Resetting passwords for accounts with access to signing certificates. + +Template responses +------------------ + +These template responses should be used as guidance for messaging +in various points in the process above. They are not required to be sent as-is, +please feel free to adapt them as needed for the current context. + +.. _submit-using-ghsa: + +Submit using GitHub Security Advisories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. highlight:: none + +:: + + Thanks for submitting this report. + We use GitHub Security Advisories for triaging vulnerability reports, + please review our security policy before submitting: + + https://devguide.python.org/security/policy/ + + Then submit your report here: + + https://github.com/python/cpython/security/advisories/new + +Rejecting a vulnerability report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + Thanks for your report. We've determined that the report doesn't constitute + a vulnerability. Let us know if you disagree with this determination. + If you are interested in working on this further, you can optionally open a + public issue on GitHub. + +Rejecting a duplicate report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + Thanks for your report. We've determined that this submission is a duplicate + of an issue that is already being tracked internally. Please refrain from + public disclosure until the issue has been resolved. + +Advisory email +~~~~~~~~~~~~~~ + +:: + + Title: [{CVE-YYYY-XXXX}] {title} + + There is a {LOW, MEDIUM, HIGH, CRITICAL} severity vulnerability affecting {project}. + + {description} + + Please see the linked CVE ID for the latest information on affected versions: + + * https://www.cve.org/CVERecord?id={CVE-YYYY-XXXX} + * {pull request URL} + +.. _psrt-members-emeritus: + +Members emeritus +---------------- + +Members who have previously served on the PSRT. + +.. csv-table:: + :header: "Name", "GitHub username", "Notes" + :file: psrt-emeritus.csv + :encoding: "utf-8" diff --git a/developer-workflow/sbom.rst b/security/sbom.rst similarity index 100% rename from developer-workflow/sbom.rst rename to security/sbom.rst diff --git a/testing/buildbots.rst b/testing/buildbots.rst index 1e711ab6bb..fc420d3f79 100644 --- a/testing/buildbots.rst +++ b/testing/buildbots.rst @@ -11,35 +11,38 @@ branches <devcycle>`, Python has a set of dedicated machines (called *buildbots* or *build workers*) used for continuous integration. They span a number of hardware/operating system combinations. Furthermore, each machine hosts several *builders*, one per active branch: when a new change is pushed -to this branch on the public `GitHub repository <https://github.com/python/cpython>`__, all corresponding builders -will schedule a new build to be run as soon as possible. +to this branch on the public `GitHub repository <https://github.com/python/cpython>`__, +all corresponding builders will schedule a new build to be run as soon as possible. -The build steps run by the buildbots are the following: +The build steps run by the buildbots are: * Check out the source tree for the change which triggered the build * Compile Python * Run the test suite using :ref:`strenuous settings <strenuous_testing>` * Clean up the build tree -It is your responsibility, as a core developer, to check the automatic -build results after you push a change to the repository. It is therefore -important that you get acquainted with the way these results are presented, +It is the responsibility of core team members to check the automatic +build results after they push a change to the repository. It is therefore +important that they are acquainted with the way these results are presented, and how various kinds of failures can be explained and diagnosed. + In case of trouble ================== Please read this page in full. If your questions aren't answered here and you need assistance with the buildbots, a good way to get help is to either: -* contact the ``python-buildbots@python.org`` mailing list where all buildbot - worker owners are subscribed; or +* contact the `python-buildbots@python.org + <https://mail.python.org/archives/list/python-buildbots@python.org/>`__ + mailing list where all buildbot worker owners are subscribed; or * contact the release manager of the branch you have issues with. + Buildbot failures on pull requests ================================== -The ``bedevere-bot`` on GitHub will put a message on your merged Pull Request +The Bedevere bot on GitHub will put a message on your merged pull request if building your commit on a stable buildbot worker fails. Take care to evaluate the failure, even if it looks unrelated at first glance. @@ -48,6 +51,7 @@ after each commit. In particular, reference leaks builds take several hours to complete so they are done periodically. This is why it's important for you to be able to check the results yourself, too. + Triggering on pull requests =========================== @@ -63,19 +67,19 @@ buildbots again on a later commit, you'll have to remove the label and add it again. If you want to test a pull request against specific platforms, you can trigger -one or more build bots by posting a comment that begins with: +one or more buildbots by posting a comment that begins with: .. code-block:: none !buildbot regex-matching-target -For example to run both the iOS and Android build bot, you can use: +For example to run both the iOS and Android buildbots, you can use: .. code-block:: none !buildbot ios|android -bedevere-bot will post a comment indicating which build bots, if +Bedevere will post a comment indicating which buildbots, if any, were matched. If none were matched, or you do not have the necessary permissions to trigger a request, it will tell you that too. @@ -83,38 +87,30 @@ The ``!buildbot`` comment will also only run buildbots on the most recent commit. To trigger the buildbots again on a later commit, you will have to repeat the comment. + Checking results of automatic builds ==================================== -There are three ways of visualizing recent build results: +The Web interface at https://buildbot.python.org/#/ has several ways of +visualizing recent build results: -* The Web interface for each branch at https://www.python.org/dev/buildbot/, - where the so-called "waterfall" view presents a vertical rundown of recent - builds for each builder. When interested in one build, you'll have to - click on it to know which commits it corresponds to. Note that - the buildbot web pages are often slow to load, be patient. +* A `release status dashboard <https://buildbot.python.org/#/release_status>`__ + that shows the status of buildbots for each active branch, + summarizing whether the builds are ready for release. -* The command-line ``bbreport.py`` client, which you can get from - https://code.google.com/archive/p/bbreport. Installing it is trivial: just add - the directory containing ``bbreport.py`` to your system path so that - you can run it from any filesystem location. For example, if you want - to display the latest build results on the development ("main") branch, - type:: +* A `waterfall view <https://buildbot.python.org/#/waterfall>`__ + that presents a vertical rundown of recent builds for each builder. + When interested in one build, click on it for more detail such as logs and + which commits it corresponds to. - bbreport.py -q 3.x - -* The buildbot "console" interface at https://buildbot.python.org/all/ - This works best on a wide, high resolution +* A `console view <https://buildbot.python.org/#/console>`__, + which works best on a wide, high resolution monitor. Clicking on the colored circles will allow you to open a new page containing whatever information about that particular build is of interest to you. You can also access builder information by clicking on the builder status bubbles in the top line. -If you like IRC, having an IRC client open to the #python-dev-notifs channel on -irc.libera.chat is useful. Any time a builder changes state (last build -passed and this one didn't, or vice versa), a message is posted to the channel. -Keeping an eye on the channel after pushing a commits is a simple way to get -notified that there is something you should look in to. +Note that the buildbot web pages are often slow to load, be patient. Some buildbots are much faster than others. Over time, you will learn which ones produce the quickest results after a build, and which ones take the @@ -124,20 +120,24 @@ Also, when several commits are pushed in a quick succession in the same branch, it often happens that a single build is scheduled for all these commits. + Stability ========= -A subset of the buildbots are marked "stable". They are taken into account -when making a new release. The rule is that all stable builders must be free of -persistent failures when the release is cut. It is absolutely **vital** -that core developers fix any issue they introduce on the stable buildbots, -as soon as possible. +A subset of the buildbots are marked as +`"stable" <https://buildbot.python.org/#/builders?tags=%2Bstable>`__. +They are taken into account when making a new release. +The rule is that all tier 1 and 2 stable builders must be free of +persistent failures when the release is cut (see :pep:`11` for more information). +It is absolutely **vital** that core team members fix or revert any issue they +introduce on the stable buildbots, as soon as possible. This does not mean that other builders' test results can be taken lightly, either. Some of them are known for having platform-specific issues that prevent some tests from succeeding (or even terminating at all), but introducing additional failures should generally not be an option. + Flags-dependent failures ======================== @@ -148,11 +148,8 @@ or to Python itself. To reproduce, make sure you use the same flags as the buildbots: they can be found out simply by clicking the **stdio** link for the failing build's tests. For example:: - ./python.exe -Wd -E -bb ./Lib/test/regrtest.py -uall -rwW + ./python -E -m test --slow-ci --timeout=1200 -j2 --junit-xml test-results.xml -j10 -.. note:: - Running ``Lib/test/regrtest.py`` is exactly equivalent to running - ``-m test``. Ordering-dependent failures =========================== @@ -170,31 +167,37 @@ run, and check the beginning of the test output proper. Let's assume, for the sake of example, that the output starts with: .. code-block:: none - :emphasize-lines: 6 + :emphasize-lines: 9 - ./python -Wd -E -bb Lib/test/regrtest.py -uall -rwW - == CPython 3.3a0 (default:22ae2b002865, Mar 30 2011, 13:58:40) [GCC 4.4.5] - == Linux-2.6.36-gentoo-r5-x86_64-AMD_Athlon-tm-_64_X2_Dual_Core_Processor_4400+-with-gentoo-1.12.14 little-endian - == /home/buildbot/buildarea/3.x.ochtman-gentoo-amd64/build/build/test_python_29628 - Testing with flags: sys.flags(debug=0, inspect=0, interactive=0, optimize=0, dont_write_bytecode=0, no_user_site=0, no_site=0, ignore_environment=1, verbose=0, bytes_warning=2, quiet=0) - Using random seed 2613169 - [ 1/353] test_augassign - [ 2/353] test_functools + ./python -E -m test --slow-ci --timeout=2400 -j2 -u-cpu,-urlfetch,-network --junit-xml test-results.xml -j4 + == CPython 3.15.0a6+ (heads/main:d625f7da33b, Feb 13 2026, 17:27:29) [GCC 12.2.0] + == Linux-6.12.20+rpt-rpi-v8-aarch64-with-glibc2.36 little-endian + == Python build: release + == cwd: /home/stan/buildarea/3.x.stan-raspbian.nondebug/build/build/test_python_worker_181905æ + == CPU count: 4 + == encodings: locale=ISO-8859-1 FS=utf-8 + == resources: all,-cpu,-network,-urlfetch + Using random seed: 1000348774 + 0:00:00 load avg: 3.34 Run 500 tests in parallel using 4 worker processes (timeout: 40 min, worker timeout: 45 min) + 0:00:01 load avg: 3.34 [ 1/500] test_colorsys passed + 0:00:01 load avg: 3.34 [ 2/500] test_float passed -You can reproduce the exact same order using:: +You can reproduce the exact same order by adding the ``--randseed 1000348774`` option:: - ./python -Wd -E -bb -m test -uall -rwW --randseed 2613169 + ./python -E -m test --slow-ci --timeout=2400 -j2 -u-cpu,-urlfetch,-network --junit-xml test-results.xml -j4 --randseed 1000348774 It will run the following sequence (trimmed for brevity): .. code-block:: none - [ 1/353] test_augassign - [ 2/353] test_functools - [ 3/353] test_bool - [ 4/353] test_contains - [ 5/353] test_compileall - [ 6/353] test_unicode + [ 1/500] test_colorsys + [ 2/500] test_float + [ 3/500] test.test_io.test_memoryio + [ 4/500] test_profile + [ 5/500] test_picklebuffer + [ 6/500] test_zipimport + [ 7/500] test_devpoll + ... If this is enough to reproduce the failure on your setup, you can then bisect the test sequence to look for the specific interference causing the @@ -202,26 +205,26 @@ failure. Copy and paste the test sequence in a text file, then use the ``--fromfile`` (or ``-f``) option of the test runner to run the exact sequence recorded in that text file:: - ./python -Wd -E -bb -m test -uall -rwW --fromfile mytestsequence.txt + ./python -E -m test --slow-ci --timeout=2400 -j2 -u-cpu,-urlfetch,-network --junit-xml test-results.xml -j4 --fromfile mytestsequence.txt -In the example sequence above, if ``test_unicode`` had failed, you would +In the example sequence above, if ``test_zipimport`` had failed, you would first test the following sequence: .. code-block:: none - [ 1/353] test_augassign - [ 2/353] test_functools - [ 3/353] test_bool - [ 6/353] test_unicode + [ 1/500] test_colorsys + [ 2/500] test_float + [ 3/500] test.test_io.test_memoryio + [ 6/500] test_zipimport And, if it succeeds, the following one instead (which, hopefully, shall fail): .. code-block:: none - [ 4/353] test_contains - [ 5/353] test_compileall - [ 6/353] test_unicode + [ 4/500] test_profile + [ 5/500] test_picklebuffer + [ 6/500] test_zipimport Then, recursively, narrow down the search until you get a single pair of tests which triggers the failure. It is very rare that such an interference @@ -243,7 +246,7 @@ not reach a perfect level of reproducibility. Some of them will sometimes display spurious failures, depending on various conditions. Here are common offenders: -* Network-related tests, such as ``test_poplib``, ``test_urllibnet``, etc. +* Network-related tests, such as ``test_poplib``, ``test_urllibnet``, and so on. Their failures can stem from adverse network conditions, or imperfect thread synchronization in the test code, which often has to run a server in a separate thread. diff --git a/testing/coverage.rst b/testing/coverage.rst index c34bb5235a..48c2efc985 100644 --- a/testing/coverage.rst +++ b/testing/coverage.rst @@ -4,13 +4,7 @@ Increase test coverage ====================== -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst Python development follows a practice that all semantic changes and additions to the language and :abbr:`stdlib (standard library)` are accompanied by diff --git a/testing/index.rst b/testing/index.rst index 55bdd3d08b..6e65c5fdf9 100644 --- a/testing/index.rst +++ b/testing/index.rst @@ -10,5 +10,6 @@ Testing and buildbots run-write-tests silence-warnings coverage + oss-fuzz buildbots new-buildbot-worker diff --git a/testing/new-buildbot-worker.rst b/testing/new-buildbot-worker.rst index a0fbf9681a..aea43a782a 100644 --- a/testing/new-buildbot-worker.rst +++ b/testing/new-buildbot-worker.rst @@ -25,7 +25,7 @@ mailing list. This mailing list is also the place to contact if you want to contribute a buildbot but have questions. As for what kind of buildbot to run...take a look at our `current fleet -<https://buildbot.python.org/all/>`__. Pretty much anything that isn't +<builders_>`__. Pretty much anything that isn't on that list would be interesting: different Linux/Unix distributions, different versions of the various OSes, other OSes if you or someone are prepared to make the test suite actually pass on that new OS. Even if you only @@ -40,14 +40,14 @@ Preparing for buildbot worker setup Since the goal is to build Python from source, the system will need to have everything required to do normal python development: a compiler, a linker, and -(except on windows) the "development" headers for any of the optional modules -(zlib, OpenSSL, etc) supported by the platform. Follow the steps outlined in +(except on Windows) the "development" headers for any of the optional modules +(zlib, OpenSSL, and so on) supported by the platform. Follow the steps outlined in :ref:`setup` for the target platform, all the way through to having a working -compiled python. +compiled Python. In order to set up the buildbot software, you will need to obtain an identifier and password for your worker so it can join the fleet. Open an issue in the -`configuration repository <https://github.com/python/buildmaster-config/issues/new/choose>`__ +`configuration repository <https://github.com/python/buildmaster-config/issues/new?template=new_worker.yml>`__ to discuss adding your worker and to obtain the needed workername and password. You can do some of the steps that follow before having the credentials, but it is easiest to have them before @@ -60,209 +60,451 @@ Setting up the buildbot worker Conventional always-on machines ------------------------------- -You need a recent version of the `buildbot <https://buildbot.net/>`__ software, -and you will probably want a separate 'buildbot' user to run the buildbot -software. You may also want to set the buildbot up using a virtual -environment, depending on how you manage your system. We won't cover how to that -here; it doesn't differ from setting up a virtual environment for any other -software, but you'll need to modify the sequence of steps below as appropriate -if you choose that path. +You need a recent version of the `buildbot <https://buildbot.net/>`__ worker +software. On most platforms the distribution's package manager provides the +``buildbot-worker`` package, which also creates a dedicated service account, +systemd unit (or equivalent), and the necessary directories. For platforms +where no package exists, ``pip install buildbot-worker`` is the fallback, but +you will need to create the service account, directories, and service unit +manually. You may also want to set the buildbot up using a virtual +environment, depending on how you manage your system; you'll need to adjust +the steps below as appropriate if you choose that path. -For Linux: +.. tab:: Linux -* If your package manager provides the buildbot worker software, that is - probably the best way to install it; it may create the buildbot user for - you, in which case you can skip the next step. Otherwise, do ``pip install - buildbot-worker`` or ``pip3 install buildbot-worker``. -* Create a ``buildbot`` user (using, eg: ``useradd``) if necessary. -* Log in as the buildbot user. + .. tab:: Fedora / RHEL / CentOS -For Mac: + **Fedora**:: -* Create a buildbot user using the OS/X control panel user admin. It - should be a "standard" user. -* Log in as the buildbot user. -* Install the buildbot worker [#]_ by running ``pip install buildbot-worker``. + dnf install buildbot-worker -For Windows: + **RHEL 8** (requires EPEL):: -* Create a buildbot user as a "standard" user. -* Install the latest version of Python from python.org. -* Open a Command Prompt. -* Execute ``python -m pip install pypiwin32 buildbot-worker`` (note that - ``python.exe`` is not added to ``PATH`` by default, making the - ``python`` command accessible is left as an exercise for the user). + subscription-manager repos --enable codeready-builder-for-rhel-8-$(arch)-rpms + dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + dnf install buildbot-worker -In a terminal window for the buildbot user, issue the following commands (you -can put the ``buildarea`` wherever you want to):: + **RHEL 9** (requires EPEL):: - mkdir buildarea - buildbot-worker create-worker buildarea buildbot-api.python.org:9020 workername workerpasswd + subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms + dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + dnf install buildbot-worker -(Note that on Windows, the ``buildbot-worker`` command will be in the -:file:`Scripts` directory of your Python installation.) + **CentOS Stream 9 / 10** (requires CRB + EPEL):: -On Windows, `the maximum length for a path is limited -<https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation>`__. -This might cause some tests to fail, unless long paths support is enabled. + dnf config-manager --set-enabled crb + dnf install epel-release epel-next-release + dnf install buildbot-worker -Use this PowerShell command to check whether long paths are enabled:: + The RPM creates a ``buildbot-worker`` system user, installs a + templated systemd unit ``buildbot-worker@.service``, and creates + ``/var/lib/buildbot/worker/`` as the base directory for worker + instances. + + .. tip:: + + If your system has most of its disk space on ``/home`` rather than + on the root partition, create the worker data under ``/home`` and + symlink it so the packaged systemd unit still works:: + + mkdir -p /home/buildbot-worker/worker + ln -s /home/buildbot-worker/worker /var/lib/buildbot/worker + + Adjust ownership and paths to match your distro's conventions. + + Create the worker (replace ``WORKERNAME`` and ``WORKERPASSWD`` with + the credentials provided to you from your buildmaster-config issue):: + + sudo -u buildbot-worker buildbot-worker create-worker \ + /var/lib/buildbot/worker/WORKERNAME \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD + + Edit ``info/admin``, ``info/host``, and ``buildbot.tac`` in the worker + directory (see below for recommended settings). + + Enable and start the service:: + + systemctl enable --now buildbot-worker@WORKERNAME.service + + .. tab:: Debian / Ubuntu + + :: + + apt install buildbot-worker + + The package creates a ``buildbot`` system user, installs a templated + systemd unit ``buildbot-worker@.service``, and creates + ``/var/lib/buildbot/workers/`` as the base directory for worker + instances. + + .. tip:: + + If your system has most of its disk space on ``/home`` rather than + on the root partition, create the worker data under ``/home`` and + symlink it so the packaged systemd unit still works:: + + mkdir -p /home/buildbot/workers + ln -s /home/buildbot/workers /var/lib/buildbot/workers + + Adjust ownership and paths to match your distro's conventions. + + Create the worker (replace ``WORKERNAME`` and ``WORKERPASSWD`` with + the credentials provided to you from your buildmaster-config issue):: + + sudo -u buildbot buildbot-worker create-worker \ + /var/lib/buildbot/workers/WORKERNAME \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD + + Edit ``info/admin``, ``info/host``, and ``buildbot.tac`` in the worker + directory (see below for recommended settings). + + Enable and start the service:: + + systemctl enable --now buildbot-worker@WORKERNAME.service + + .. tab:: Other / pip + + For distros without a ``buildbot-worker`` package, install via pip:: + + pip install buildbot-worker + + **NixOS** users should use the built-in ``services.buildbot-worker`` + NixOS module; see the + `nixpkgs module source <https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/continuous-integration/buildbot/worker.nix>`__ + for available options. + + **Arch Linux** has buildbot packages in the AUR, but they are + currently unmaintained. Using pip is more reliable. + + pip does **not** create a system user, directories, or service unit. + Set these up manually. On distros with ``useradd``:: + + useradd --system --shell /sbin/nologin \ + --home-dir /var/lib/buildbot/worker --create-home buildbot-worker + + On Alpine Linux (BusyBox):: + + adduser -S -D -H -h /var/lib/buildbot/worker -s /sbin/nologin buildbot-worker + + Then create the directories:: + + mkdir -p /var/lib/buildbot/worker + chown buildbot-worker:buildbot-worker /var/lib/buildbot/worker + + Create the worker (replace ``WORKERNAME`` and ``WORKERPASSWD`` with + the credentials provided to you from your buildmaster-config issue):: + + sudo -u buildbot-worker buildbot-worker create-worker \ + /var/lib/buildbot/worker/WORKERNAME \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD + + Edit ``info/admin``, ``info/host``, and ``buildbot.tac`` in the worker + directory (see below for recommended settings). + + On systemd-based distros, a service unit must also be installed; see + the service management section below. + + +.. tab:: Unix + + .. tab:: FreeBSD + + :: + + pkg install devel/py-buildbot-worker + + The package creates a ``buildbot`` system user, installs an + ``rc.d`` service with profile support, and creates + ``/var/db/buildbot/workers/`` as the base directory for worker + instances. + + Create the worker (replace ``WORKERNAME`` and ``WORKERPASSWD`` with + the credentials provided to you from your buildmaster-config issue):: + + su -m buildbot -c "buildbot-worker create-worker \ + /var/db/buildbot/workers/WORKERNAME \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD" + + Edit ``info/admin``, ``info/host``, and ``buildbot.tac`` in the worker + directory (see below for recommended settings). + + Enable and start the service. The rc.d script uses profile names as + shell variable identifiers, so pick a short name without hyphens + (it does not need to match the worker name):: + + sysrc buildbot_worker_enable=YES + sysrc buildbot_worker_profiles="myworker" + sysrc buildbot_worker_myworker_enable=YES + sysrc buildbot_worker_myworker_basedir=/var/db/buildbot/workers/WORKERNAME + service buildbot-worker start + + .. tab:: OpenBSD + + :: + + pkg_add buildbot-worker + + The package creates a ``_buildslave`` system user, installs an + ``rc.d`` service, and creates ``/var/buildslave/`` as the default + worker directory. + + Create the worker (replace ``WORKERNAME`` and ``WORKERPASSWD`` with + the credentials provided to you from your buildmaster-config issue):: + + su -m _buildslave -c "buildbot-worker create-worker \ + /var/buildslave \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD" + + Edit ``info/admin``, ``info/host``, and ``buildbot.tac`` in the worker + directory (see below for recommended settings). + + Enable and start the service:: + + rcctl enable buildbot_worker + rcctl start buildbot_worker + + The ``rc.d`` script supports a single worker. To run multiple + workers, create each in a subdirectory and point the service flags + at the desired one (or create additional ``rc.d`` scripts):: + + su -m _buildslave -c "buildbot-worker create-worker \ + /var/buildslave/WORKERNAME \ + buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD" + rcctl enable buildbot_worker + rcctl set buildbot_worker flags /var/buildslave/WORKERNAME + rcctl start buildbot_worker + + +.. tab:: macOS + + * Create a buildbot user using the macOS control panel user admin. It + should be a "standard" user. + * Log in as the buildbot user. + * Install the buildbot worker [#]_ by running ``pip install buildbot-worker``. + + In a terminal window for the buildbot user, issue the following commands (you + can put the ``buildarea`` wherever you want to):: + + mkdir buildarea + buildbot-worker create-worker buildarea buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD + + +.. tab:: Windows + + * Create a buildbot user as a "standard" user. + * Install the latest version of Python from python.org. + * Open a Command Prompt. + * Execute ``python -m pip install pywin32 buildbot-worker`` (note that + ``python.exe`` is not added to ``PATH`` by default, making the + ``python`` command accessible is left as an exercise for the user). + + On Windows, `the maximum length for a path is limited + <https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation>`__. + This might cause some tests to fail, unless long paths support is enabled. + + Use this PowerShell command to check whether long paths are enabled:: Get-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -If the value is not "1", you can enable long paths using this PowerShell command:: + If the value is not "1", you can enable long paths using this PowerShell command:: New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force -Once this initial worker setup completes, you should edit the files -``buildarea/info/admin`` and ``buildarea/info/host`` to provide your contact -info and information on the host configuration, respectively. This information -will be presented in the buildbot web pages that display information about the -builders running on your worker. + + In a terminal window for the buildbot user, issue the following commands. + (You can put the ``buildarea`` wherever you want to.) + Note that the ``buildbot-worker`` command will be in the + :file:`Scripts` directory of your Python installation. Here and in the rest + of the guide, you may need to launch it using the full path. + + .. code-block:: + + mkdir buildarea + buildbot-worker create-worker buildarea buildbot-api.python.org:9020 WORKERNAME WORKERPASSWD + + +The ``info/admin`` file in the worker directory should contain your contact +information, and ``info/host`` should describe the host configuration. This +information is displayed on the buildbot web interface. Since these pages are +publicly visible, consider obfuscating your email address (for example, +``user AT example.com``) to avoid spam from web scrapers. + +The recommended ``buildbot.tac`` settings are: + +* ``keepalive = 60`` -- the buildmaster uses a 60-second keepalive interval; + the default of ``600`` is too high and can cause spurious disconnections. +* ``delete_leftover_dirs = 1`` -- automatically cleans up build directories + that the master no longer needs. + +.. tip:: + + Build directories and ``twistd.log`` rotations can accumulate over time. + Monitor free disk space on the partition that holds the worker directory, + even with ``delete_leftover_dirs`` enabled. + +Service management +~~~~~~~~~~~~~~~~~~ You will also want to make sure that the worker is started when the machine reboots: -For Linux: +.. tab:: Linux + + .. tab:: Distro package + + If you installed via a distro package (Fedora, RHEL, CentOS, Debian, + or Ubuntu), the service was already enabled in the installation + step above. -* For systemd based distributions, you can create a systemd unit file in order - to manage the service. Create the unit file named ``buildbot-worker.service`` - under ``/home/buildbot/.config/systemd/user/`` and change the paths according to where the - buildbot-worker binary resides. You can verify its location by running - ``which buildbot-worker``. - If you installed the buildbot-worker through - your package manager it would be:: - - [Unit] - Description=Buildbot Worker - Wants=network-online.target - After=network-online.target local-fs.target - - [Service] - Type=forking - PIDFile=/home/buildbot/buildarea/twistd.pid - WorkingDirectory=/home/buildbot/buildarea - ExecStart=/usr/bin/buildbot-worker start - ExecReload=/usr/bin/buildbot-worker restart - ExecStop=/usr/bin/buildbot-worker stop - Restart=always - User=buildbot - - [Install] - WantedBy=multi-user.target - - - If you installed the buildbot-worker through pip, the systemd unit - file should look like this:: - - [Unit] - Description=Buildbot Worker - Wants=network-online.target - After=network-online.target local-fs.target - - [Service] - Type=forking - PIDFile=/home/buildbot/buildarea/twistd.pid - WorkingDirectory=/home/buildbot/buildarea - ExecStart=/usr/local/bin/buildbot-worker start - ExecReload=/usr/local/bin/buildbot-worker restart - ExecStop=/usr/local/bin/buildbot-worker stop - Restart=always - User=buildbot - - [Install] - WantedBy=multi-user.target - - - Then enable lingering for the buildbot user via the - ``loginctl enable-linger buildbot`` command and you can start - the service through a login shell of the buildbot user - via the ``systemctl --user enable --now buildbot-worker.service`` - command. - - Note that using a systemd unit file, might produce some selinux warnings on systems - where the enforcing mode is enabled, usually related to the twistd.pid file. - If the service fails to start, you should check the output of - ``systemctl status buildbot-worker.service`` as well as the - ``/var/log/audit/audit.log`` file (e.g. through - ``sealert -a /var/log/audit/audit.log``) for potential issues and remedies. - - -* Alternatively you can create a cronjob. Add the following line to ``/etc/crontab``:: - - @reboot buildbot-worker restart /path/to/buildarea - - Note that we use ``restart`` rather than ``start`` in case a crash has - left a ``twistd.pid`` file behind. - -For OSX: - -* Create a bin directory for your buildbot user:: - - mkdir bin - -* Place the following script, named ``run_worker.sh``, into that directory:: - - #!/bin/bash - export PATH=/usr/local/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH - export LC_CTYPE=en_US.utf-8 - cd /Users/buildbot/buildarea - twistd --nodaemon --python=buildbot.tac --logfile=buildbot.log --prefix=worker - - If you use pip with Apple's system python, add '/System' to the front of - the path to the Python bin directory. - -* Place a file with the following contents into ``/Library/LaunchDaemons``: - - .. code-block:: xml - - <?xml version="1.0" encoding="UTF-8"?> - <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" - "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> - <plist version="1.0"> - <dict> - <key>Label</key> - <string>net.buildbot.worker</string> - <key>UserName</key> - <string>buildbot</string> - <key>WorkingDirectory</key> - <string>/Users/buildbot/buildarea</string> - <key>ProgramArguments</key> - <array> - <string>/Users/buildbot/bin/run_worker.sh</string> - </array> - <key>StandardOutPath</key> - <string>twistd.log</string> - <key>StandardErrorPath</key> - <string>twistd.log</string> - <key>KeepAlive</key> - <true/> - <key>SessionCreate</key> - <true/> - </dict> - </plist> + .. tab:: Manual systemd - The recommended name for the file is ``net.buildbot.worker``. + If you installed via pip, you need to install a systemd unit yourself. + The upstream buildbot project provides a + `contributed template unit <https://github.com/buildbot/buildbot/blob/master/worker/contrib/systemd/buildbot-worker%40.service>`__ + along with + `sysusers.d and tmpfiles.d configs <https://github.com/buildbot/buildbot/tree/master/common/contrib/systemd>`__. -For Windows: + Create ``/etc/systemd/system/buildbot-worker@.service`` with the + following contents:: -* Add a Scheduled Task to run ``buildbot-worker start buildarea`` as the - buildbot user "when the computer starts up". It is best to provide - absolute paths to the ``buildbot-worker`` command and the :file:`buildarea` - directory. It is also recommended to set the task to run in the - directory that contains the :file:`buildarea` directory. + [Unit] + Description=Buildbot Worker %i + Documentation=man:buildbot-worker(1) https://docs.buildbot.net/ + After=network.target + ConditionDirectoryNotEmpty=/var/lib/buildbot/worker/%i + ConditionFileNotEmpty=/var/lib/buildbot/worker/%i/buildbot.tac -* Alternatively (note: don't do both!), set up the worker - service as described in the `buildbot documentation - <https://docs.buildbot.net/current/manual/installation/misc.html#launching-worker-as-windows-service>`__. - -To start the worker running for your initial testing, you can do:: + [Service] + Type=simple + User=buildbot-worker + Group=buildbot-worker + WorkingDirectory=/var/lib/buildbot/worker/ + StateDirectory=buildbot/worker + ExecStart=/usr/local/bin/buildbot-worker start --nodaemon %i + Restart=always + ProtectSystem=full + ProtectHome=yes + PrivateDevices=yes + PrivateTmp=yes - buildbot-worker start buildarea + [Install] + WantedBy=multi-user.target + + Adjust ``User``, ``Group``, ``WorkingDirectory``, and the + ``ExecStart`` path to match your setup. If your worker data is + symlinked from ``/home`` (see the filesystem layout tip above), + change ``ProtectHome=yes`` to ``ProtectHome=no`` so systemd can + follow the symlink. Then:: + + systemctl daemon-reload + systemctl enable --now buildbot-worker@WORKERNAME.service + + .. tab:: SysV init + + For distros without systemd (such as Alpine Linux with OpenRC), + upstream provides a + `SysV init script <https://github.com/buildbot/buildbot/blob/master/worker/contrib/init-scripts/buildbot-worker.init.sh>`__ + with a + `default configuration file <https://github.com/buildbot/buildbot/blob/master/worker/contrib/init-scripts/buildbot-worker.default>`__. + Install these as ``/etc/init.d/buildbot-worker`` and + ``/etc/default/buildbot-worker`` respectively, then configure the + worker instances in the default file. + + .. tab:: cron job + + If neither systemd nor a SysV init script is practical, you can use + a cron job. Add the following line to ``/etc/crontab``:: + + @reboot buildbot-worker restart /path/to/workerdir + + Note that ``restart`` is used rather than ``start`` in case a crash + has left a ``twistd.pid`` file behind. + +.. tab:: Unix + + If you installed via a package on FreeBSD or OpenBSD, the service + was already enabled in the installation step above. To manage it + manually: + + On FreeBSD:: + + service buildbot-worker status + service buildbot-worker restart + + On OpenBSD:: + + rcctl check buildbot_worker + rcctl restart buildbot_worker + + If you installed via pip, you will need to write an ``rc.d`` script + or use the cron job approach described in the Linux tab. + +.. tab:: macOS + + * Create a bin directory for your buildbot user:: + + mkdir bin + + * Place the following script, named ``run_worker.sh``, into that directory:: + + #!/bin/bash + export PATH=/usr/local/bin:/Library/Frameworks/Python.framework/Versions/Current/bin:$PATH + export LC_CTYPE=en_US.utf-8 + cd /Users/buildbot/buildarea + twistd --nodaemon --python=buildbot.tac --logfile=buildbot.log --prefix=worker + + * Place a file with the following contents into ``/Library/LaunchDaemons``: + + .. code-block:: xml + + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" + "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> + <plist version="1.0"> + <dict> + <key>Label</key> + <string>net.buildbot.worker</string> + <key>UserName</key> + <string>buildbot</string> + <key>WorkingDirectory</key> + <string>/Users/buildbot/buildarea</string> + <key>ProgramArguments</key> + <array> + <string>/Users/buildbot/bin/run_worker.sh</string> + </array> + <key>StandardOutPath</key> + <string>twistd.log</string> + <key>StandardErrorPath</key> + <string>twistd.log</string> + <key>KeepAlive</key> + <true/> + <key>SessionCreate</key> + <true/> + </dict> + </plist> + + The recommended name for the file is ``net.buildbot.worker``. + +.. tab:: Windows + + * Add a Scheduled Task to run ``buildbot-worker start buildarea`` as the + buildbot user "when the computer starts up". It is best to provide + absolute paths to the ``buildbot-worker`` command and the :file:`buildarea` + directory. It is also recommended to set the task to run in the + directory that contains the :file:`buildarea` directory. + + * Alternatively (note: don't do both!), set up the worker + service as described in the `buildbot documentation + <https://docs.buildbot.net/current/manual/installation/misc.html#launching-worker-as-windows-service>`__. + +If you have not already started the worker through a service manager, you +can start it manually for initial testing:: + + buildbot-worker start /path/to/workerdir Then you can either wait for someone to make a commit, or you can pick a builder associated with your worker from the `list of builders -<https://buildbot.python.org/all/>`__ and force a build. +<builders_>`__ and force a build. In any case you should initially monitor builds on your builders to make sure the tests are passing and to resolve any platform issues that may be revealed @@ -270,13 +512,7 @@ by tests that fail. Unfortunately we do not currently have a way to notify you only of failures on your builders, so doing periodic spot checks is also a good idea. -.. note:: - If your buildbot worker is disconnecting regularly, it may be a symptom of the - default ``keepalive`` value (``600`` for 10 minutes) being `set - <https://docs.buildbot.net/latest/manual/installation/worker.html#cmdoption-buildbot-worker-create-worker-keepalive>`__ - too high. You can change it to a lower value (for example, ``180`` for 3 minutes) - in the ``buildbot.tac`` file found in your build area. - +---- Latent workers -------------- @@ -339,7 +575,7 @@ Necessary tasks include, obviously, keeping the buildbot running. Currently the system for notifying buildbot owners when their workers go offline is not working; this is something we hope to resolve. So currently it is helpful if you periodically check the status of your worker. We will also contact you -via your contact address in ``buildarea/info/admin`` when we notice there is a +via your contact address in ``info/admin`` when we notice there is a problem that has not been resolved for some period of time and you have not responded to a posting on the python-buildbots list about it. @@ -368,19 +604,19 @@ a table listing all of the outbound ports used by the buildbot and the python test suite (this list may not be complete as new tests may have been added since this table was last vetted): -======= =================== ================================================ -Port Host Description -======= =================== ================================================ -20, 21 ftp.debian.org test_urllib2net -53 your DNS server test_socket, and others implicitly -80 python.org (several tests) +======= ========================== ================================================ +Port Host Description +======= ========================== ================================================ +20, 21 ftp.debian.org test_urllib2net +53 your DNS server test_socket, and others implicitly +80 python.org (several tests) example.com -119 news.gmane.org test_nntplib -443 (various) test_ssl -465 smtp.gmail.com test_smtpnet -587 smtp.gmail.com test_smtpnet -9020 python.org connection to buildmaster -======= =================== ================================================ +119 news.gmane.org test_nntplib (Python versions < 3.13) +443 (various) test_ssl +465 smtp.gmail.com test_smtpnet +587 smtp.gmail.com test_smtpnet +9020 buildbot-api.python.org connection to buildmaster +======= ========================== ================================================ Many tests will also create local TCP sockets and connect to them, usually using either ``localhost`` or ``127.0.0.1``. @@ -398,10 +634,10 @@ buildbot are at least: * 512 MB RAM * 30 GB free disk space -The bigmem tests won't run in this configuration, since they require -substantially more memory, but these resources should be sufficient to ensure -that Python compiles correctly on the platform and can run the rest of the test -suite. +Many tests won't run in this configuration, since they require +substantially more memory, but these resources should be sufficient. +Builders with minimal settings might need more maintenance: they check +that Python's resource-hungry tests are tagged and skipped correctly. Security considerations @@ -413,7 +649,7 @@ This means that the code your buildbot will run will have been vetted by a commi However, mistakes and bugs happen, as could a compromise, so keep this in mind when siting your buildbot on your network and establishing the security around it. Treat the buildbot like you would any resource that is public facing and might -get hacked (use a VM and/or jail/chroot/solaris zone, put it in a DMZ, etc). +get hacked (use a VM and/or jail/chroot/solaris zone, put it in a DMZ, and so on). While the buildbot does not have any ports open for inbound traffic (and is not public facing in that sense), committer mistakes do happen, and security flaws are discovered in both released and unreleased code, so treating the buildbot @@ -422,7 +658,7 @@ as if it were fully public facing is a good policy. Code runs differently as privileged and unprivileged users. We would love to have builders running as privileged accounts, but security considerations do make that difficult, as access to root can provide access to surprising -resources (such as spoofed IP packets, changes in MAC addresses, etc) even on a +resources (such as spoofed IP packets, changes in MAC addresses, and so on) even on a VM setup. But if you are confident in your setup, we'd love to have a buildbot that runs python as root. @@ -435,3 +671,5 @@ useful as a point of reference. .. [#] If the buildbot is going to do Framework builds, it is better to use the Apple-shipped Python so as to avoid any chance of the buildbot picking up components from the installed python.org python. + +.. _builders: https://buildbot.python.org/#/builders diff --git a/testing/oss-fuzz.rst b/testing/oss-fuzz.rst new file mode 100644 index 0000000000..ec2449a4fd --- /dev/null +++ b/testing/oss-fuzz.rst @@ -0,0 +1,55 @@ +OSS-Fuzz for CPython +==================== + +CPython uses `OSS-Fuzz <https://google.github.io/oss-fuzz/>`__, Google's +continuous fuzzing service for open-source projects, to find bugs and +security vulnerabilities by feeding semi-random data to various APIs. + +CPython has two OSS-Fuzz projects: + +* `cpython3 <https://github.com/google/oss-fuzz/tree/master/projects/cpython3>`__: + The fuzz targets, seed corpora, and dictionaries can be found in the + :cpy-file:`Modules/_xxtestfuzz/` directory of CPython. This project + is maintained for existing fuzz targets; add new targets to + ``python3-libraries``. + +* `python3-libraries <https://github.com/google/oss-fuzz/tree/master/projects/python3-libraries>`__: + The fuzz targets, seed corpora, and dictionaries can be found in the + :github:`python/library-fuzzers` repository. Access to the repository is + managed through the `@python/fuzzers + <https://github.com/orgs/python/teams/fuzzers>`__ team on GitHub. + +OSS-Fuzz bug reports are private when filed, so access to crash details and +reproducer test cases is limited to those listed in the ``auto_ccs`` fields of +the OSS-Fuzz project configuration files. Those listed can log into +https://oss-fuzz.com/ with their Google account to view crash details, +reproducer test cases, and project statistics. +If you need access, contact the ``@python/fuzzers`` team. +Completed issues, and issues that remain unresolved after 90 days, are publicly +visible in the `OSS-Fuzz issue tracker +<https://issues.oss-fuzz.com/issues?q=(python3-libraries%20%7C%20cpython3)>`__. + +Coverage and target statistics are available in the OSS-Fuzz Introspector +project profiles for `cpython3 <https://introspector.oss-fuzz.com/project-profile?project=cpython3>`__ and +`python3-libraries <https://introspector.oss-fuzz.com/project-profile?project=python3-libraries>`__. + +In addition, `CIFuzz <https://google.github.io/oss-fuzz/getting-started/continuous-integration/>`__ +runs the fuzz targets on GitHub Actions for PRs to the ``main`` branch changing +relevant files. + +.. seealso:: + + The `libFuzzer <https://llvm.org/docs/LibFuzzer.html>`__ documentation for + details about the fuzzing engine used by OSS-Fuzz. + + +Adding new targets +------------------ + +Add new targets to the ``python3-libraries`` project. For more +information, see the documentation in the :github:`python/library-fuzzers` +repository. + +If the new target covers a standard library module, update the relevant CIFuzz +path configuration so pull requests touching that module trigger fuzzing. See +the ``LIBRARY_FUZZER_PATHS`` set in :cpy-file:`Tools/build/compute-changes.py`. diff --git a/testing/run-write-tests.rst b/testing/run-write-tests.rst index 34101bd2d2..405d10f039 100644 --- a/testing/run-write-tests.rst +++ b/testing/run-write-tests.rst @@ -5,13 +5,7 @@ Running and writing tests ========================= -.. raw:: html - - <script> - document.addEventListener('DOMContentLoaded', function() { - activateTab(getOS()); - }); - </script> +.. include:: /include/activate-tab.rst .. note:: diff --git a/triage/issue-tracker.rst b/triage/issue-tracker.rst index 4dd0815e4c..31673d9435 100644 --- a/triage/issue-tracker.rst +++ b/triage/issue-tracker.rst @@ -124,7 +124,8 @@ Disagreement with a resolution on the issue tracker As humans, we will have differences of opinions from time to time. First and foremost, please be respectful that care, thought, and volunteer time went into -the resolution. +the resolution. Keep in mind that contributors come from many different cultural +and linguistic backgrounds; see :ref:`multicultural-communication`. With this in mind, take some time to consider any comments made in association with the resolution of the issue. On reflection, the resolution steps may seem diff --git a/versions.rst b/versions.rst index 3d5fdced91..228b445d97 100644 --- a/versions.rst +++ b/versions.rst @@ -27,8 +27,6 @@ Dates shown in *italic* are scheduled and can be adjusted. :width: 100% :file: include/branches.csv -.. Remember to update main branch in the paragraph above too - Unsupported versions ==================== @@ -36,7 +34,7 @@ Unsupported versions .. csv-table:: :header-rows: 1 :width: 100% - :file: _static/end-of-life.csv + :file: include/end-of-life.csv .. _full-chart: @@ -47,6 +45,7 @@ Full chart .. raw:: html :file: _static/release-cycle-all.svg +.. _version-status-key: Status key ==========