diff --git a/.editorconfig b/.editorconfig index c5f3028c5036..d7fe7ad5e244 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,rs,py,sh}] indent_size = 4 -# .cirrus.yml, etc. +# ci.yml, etc. [*.yml] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index c9cf4a7d9cd0..25303e742a76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ src/clientversion.cpp export-subst +src/CMakeLists.txt export-subst diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml deleted file mode 100644 index 2a486b3f2b43..000000000000 --- a/.github/ISSUE_TEMPLATE/good_first_issue.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Good First Issue -description: (Regular devs only) Suggest a new good first issue -labels: [good first issue] -body: - - type: markdown - attributes: - value: | - Please add the label "good first issue" manually before or after opening - - A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution - - Motivate the issue and explain the solution briefly - - type: textarea - id: motivation - attributes: - label: Motivation - description: Motivate the issue - validations: - required: true - - type: textarea - id: solution - attributes: - label: Possible solution - description: Describe a possible solution - validations: - required: false - - type: textarea - id: useful-skills - attributes: - label: Useful Skills - description: For example, “`std::thread`”, “Qt6 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. - value: | - * Compiling Bitcoin Core from source - * Running the C++ unit tests and the Python functional tests - * ... - - type: textarea - attributes: - label: Guidance for new contributors - description: Please leave this to automatically add the footer for new contributors - value: | - Want to work on this issue? - - For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. - diff --git a/.github/actions/restore-caches/action.yml b/.github/actions/cache/restore/action.yml similarity index 77% rename from .github/actions/restore-caches/action.yml rename to .github/actions/cache/restore/action.yml index 21f2807f4c72..2cc5b53a1e1f 100644 --- a/.github/actions/restore-caches/action.yml +++ b/.github/actions/cache/restore/action.yml @@ -1,43 +1,51 @@ name: 'Restore Caches' description: 'Restore ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: - name: Restore Ccache cache id: ccache-cache - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} restore-keys: | ccache-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore depends sources cache id: depends-sources - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-sources-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore built depends cache id: depends-built - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} restore-keys: | depends-built-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: Restore previous releases cache id: previous-releases - uses: cirruslabs/cache/restore@v5 + uses: ./.github/actions/cache/restore/internal with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} restore-keys: | previous-releases-${{ env.CONTAINER_NAME }}- + provider: ${{ inputs.provider }} - name: export cache hits shell: bash diff --git a/.github/actions/cache/restore/internal/action.yml b/.github/actions/cache/restore/internal/action.yml new file mode 100644 index 000000000000..43dd206345df --- /dev/null +++ b/.github/actions/cache/restore/internal/action.yml @@ -0,0 +1,43 @@ +name: 'Cache Restore' +description: 'Restore a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to restore' + required: true + key: + description: 'An explicit key for restoring the cache' + required: true + restore-keys: + description: 'An ordered multiline string listing prefix-matched restore keys' + required: false + default: '' + provider: + description: 'The cache provider to use' + required: true +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.warp.outputs.cache-hit || steps.gha.outputs.cache-hit }} + cache-primary-key: + description: 'The primary key used to restore the cache' + value: ${{ steps.warp.outputs.cache-primary-key || steps.gha.outputs.cache-primary-key }} +runs: + using: 'composite' + steps: + - name: Restore cache with WarpBuild + id: warp + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/restore@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} + + - name: Restore cache with GitHub Actions + id: gha + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/restore@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: ${{ inputs.restore-keys }} diff --git a/.github/actions/save-caches/action.yml b/.github/actions/cache/save/action.yml similarity index 79% rename from .github/actions/save-caches/action.yml rename to .github/actions/cache/save/action.yml index 3072ab3f224a..5c543b3f9e78 100644 --- a/.github/actions/save-caches/action.yml +++ b/.github/actions/cache/save/action.yml @@ -1,5 +1,9 @@ name: 'Save Caches' description: 'Save ccache, depends sources, and built depends caches' +inputs: + provider: + description: 'The cache provider to use' + required: true runs: using: 'composite' steps: @@ -11,29 +15,33 @@ runs: echo "previous releases direct cache hit to primary key: ${{ env.previous-releases-cache-hit }}" - name: Save Ccache cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) }} with: path: ${{ env.CCACHE_DIR }} key: ccache-${{ env.CONTAINER_NAME }}-${{ github.run_id }} + provider: ${{ inputs.provider }} - name: Save depends sources cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-sources-cache-hit != 'true') }} with: path: ${{ env.SOURCES_PATH }} key: depends-sources-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save built depends cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.depends-built-cache-hit != 'true' )}} with: path: ${{ env.BASE_CACHE }} key: depends-built-${{ env.CONTAINER_NAME }}-${{ env.DEPENDS_HASH }} + provider: ${{ inputs.provider }} - name: Save previous releases cache - uses: cirruslabs/cache/save@v5 + uses: ./.github/actions/cache/save/internal if: ${{ (github.event_name == 'push') && (github.ref_name == github.event.repository.default_branch) && (env.previous-releases-cache-hit != 'true' )}} with: path: ${{ env.PREVIOUS_RELEASES_DIR }} key: previous-releases-${{ env.CONTAINER_NAME }}-${{ env.PREVIOUS_RELEASES_HASH }} + provider: ${{ inputs.provider }} diff --git a/.github/actions/cache/save/internal/action.yml b/.github/actions/cache/save/internal/action.yml new file mode 100644 index 000000000000..12f4156a072f --- /dev/null +++ b/.github/actions/cache/save/internal/action.yml @@ -0,0 +1,28 @@ +name: 'Cache Save' +description: 'Save a cache with WarpBuild on Warp runners and GitHub Actions cache otherwise' +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache' + required: true + key: + description: 'An explicit key for saving the cache' + required: true + provider: + description: 'The cache provider to use' + required: true +runs: + using: 'composite' + steps: + - name: Save cache with WarpBuild + if: ${{ inputs.provider == 'warp' }} + uses: WarpBuilds/cache/save@v1 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + + - name: Save cache with GitHub Actions + if: ${{ inputs.provider == 'gha' }} + uses: actions/cache/save@v5 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} diff --git a/.github/actions/configure-docker/action.yml b/.github/actions/configure-docker/action.yml index b4abf7c274ab..074b47ce348b 100644 --- a/.github/actions/configure-docker/action.yml +++ b/.github/actions/configure-docker/action.yml @@ -1,27 +1,27 @@ name: 'Configure Docker' description: 'Set up Docker build driver and configure build cache args' inputs: - cache-provider: - description: 'gha or cirrus cache provider' - required: true + provider: + description: 'The cache provider to use' + required: false + default: 'gha' runs: using: 'composite' steps: - - name: Check inputs - shell: python - run: | - # We expect only gha or cirrus as inputs to cache-provider - if "${{ inputs.cache-provider }}" not in ("gha", "cirrus"): - print("::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}") - - - name: Set up Docker Buildx + - name: Set up Docker Buildx for Warp cache + if: ${{ inputs.provider == 'warp' }} uses: docker/setup-buildx-action@v4 with: - # Use host network to allow access to cirrus gha cache running on the host driver-opts: | network=host - # This is required to allow buildkit to access the actions cache + - name: Set up Docker Buildx + if: ${{ inputs.provider != 'warp' }} + uses: docker/setup-buildx-action@v4 + + # This is required when using the gha cache backend with a manual docker buildx invocation. + # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN + # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - name: Expose actions cache variables uses: actions/github-script@v8 with: @@ -36,25 +36,18 @@ runs: - name: Construct docker build cache args shell: bash run: | - # Configure docker build cache backend - # - # On forks the gha cache will work but will use Github's cache backend. - # Docker will check for variables $ACTIONS_CACHE_URL, $ACTIONS_RESULTS_URL and $ACTIONS_RUNTIME_TOKEN - # which are set automatically when running on GitHub infra: https://docs.docker.com/build/cache/backends/gha/#synopsis - - # Use cirrus cache host - if [[ ${{ inputs.cache-provider }} == 'cirrus' ]]; then - url_args="url=${CIRRUS_CACHE_HOST},url_v2=${CIRRUS_CACHE_HOST}" - else - url_args="" + cache_options="scope=${CONTAINER_NAME}" + if [[ "${{ inputs.provider }}" == "warp" ]]; then + cache_options="url=http://127.0.0.1:49160/,version=1,${cache_options}" fi + # Configure docker build cache backend # Always optimistically --cache‑from in case a cache blob exists - args=(--cache-from "type=gha${url_args:+,${url_args}},scope=${CONTAINER_NAME}") + args=(--cache-from "type=gha,${cache_options}") - # Only add --cache-to when using the Cirrus cache provider and pushing to the default branch. - if [[ ${{ inputs.cache-provider }} == 'cirrus' && ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then - args+=(--cache-to "type=gha${url_args:+,${url_args}},mode=max,ignore-error=true,scope=${CONTAINER_NAME}") + # Only add --cache-to when pushing to the default branch. + if [[ ${{ github.event_name }} == "push" && ${{ github.ref_name }} == ${{ github.event.repository.default_branch }} ]]; then + args+=(--cache-to "type=gha,mode=max,ignore-error=true,${cache_options}") fi # Always `--load` into docker images (needed when using the `docker-container` build driver). diff --git a/.github/ci-test-each-commit-exec.py b/.github/ci-test-each-commit-exec.py index aed1526b6436..402e73527be1 100755 --- a/.github/ci-test-each-commit-exec.py +++ b/.github/ci-test-each-commit-exec.py @@ -40,8 +40,8 @@ def main(): "-DCMAKE_BUILD_TYPE=Debug", "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", "--preset=dev-mode", - # Tolerate unused member functions in intermediate commits in a pull request - "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function", + # Tolerate unused (member) functions in intermediate commits in a pull request + "-DCMAKE_CXX_FLAGS=-Wno-error=unused-member-function -Wno-error=unused-function", ]) if run(["cmake", "--build", build_dir, "-j", str(num_procs)], check=False).returncode != 0: diff --git a/.github/ci-windows-cross.py b/.github/ci-windows-cross.py index 6453cb9eac02..bf13f81ac75c 100755 --- a/.github/ci-windows-cross.py +++ b/.github/ci-windows-cross.py @@ -96,7 +96,7 @@ def run_functional_tests(): "--jobs", num_procs, "--quiet", - f"--tmpdirprefix={workspace}", + f"--tmpdirprefix={workspace / '_ _'}", "--combinedlogslen=99999999", *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), # feature_unsupported_utxo_db.py fails on Windows because of emojis in the test data directory. diff --git a/.github/ci-windows.py b/.github/ci-windows.py index a808812e408f..2635682266f5 100755 --- a/.github/ci-windows.py +++ b/.github/ci-windows.py @@ -34,8 +34,6 @@ def run(cmd, **kwargs): "fuzz": [ "-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON", "-DVCPKG_MANIFEST_FEATURES=wallet", - "-DBUILD_GUI=OFF", - "-DWITH_ZMQ=OFF", "-DBUILD_FOR_FUZZING=ON", "-DCMAKE_COMPILE_WARNING_AS_ERROR=ON", ], @@ -72,6 +70,11 @@ def generate(ci_type): "build", "-Werror=dev", "--preset=vs2026", + # Using x64-windows-release for both host and target triplets + # to ensure vcpkg builds only release packages, thereby optimizing + # build time. + # See https://github.com/microsoft/vcpkg/issues/50927. + "-DVCPKG_HOST_TRIPLET=x64-windows-release", "-DVCPKG_TARGET_TRIPLET=x64-windows-release", ] + GENERATE_OPTIONS[ci_type] if run(command, check=False).returncode != 0: @@ -193,7 +196,7 @@ def run_tests(ci_type): "--jobs", num_procs, "--quiet", - f"--tmpdirprefix={workspace}", + f"--tmpdirprefix={workspace / '_ _'}", "--combinedlogslen=99999999", *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()), ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f12ceb012ec4..434b5e16718e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,7 @@ concurrency: env: CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CIRRUS_CACHE_HOST: http://127.0.0.1:12321/ # When using Cirrus Runners this host can be used by the docker `gha` build cache type. - REPO_USE_CIRRUS_RUNNERS: 'bitcoin/bitcoin' # Use cirrus runners and cache for this repo, instead of falling back to the slow GHA runners + REPO_USE_WARP_RUNNERS: 'bitcoin/bitcoin' # Use warp runners for this repo, instead of falling back to the slow GHA runners defaults: run: @@ -47,9 +46,9 @@ jobs: fi - id: runners run: | - if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then - echo "provider=cirrus" >> "$GITHUB_OUTPUT" - echo "::notice title=Runner Selection::Using Cirrus Runners" + if [[ "${REPO_USE_WARP_RUNNERS}" == "${{ github.repository }}" ]]; then + echo "provider=warp" >> "$GITHUB_OUTPUT" + echo "::notice title=Runner Selection::Using Warp Runners" else echo "provider=gha" >> "$GITHUB_OUTPUT" echo "::notice title=Runner Selection::Using GitHub-hosted runners" @@ -58,9 +57,9 @@ jobs: test-each-commit: name: 'test ancestor commits' needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-8x' || 'ubuntu-latest' }} env: - TEST_RUNNER_PORT_MIN: "14000" # Use a larger port, to avoid colliding with CIRRUS_CACHE_HOST port 12321. + TEST_RUNNER_PORT_MIN: "14000" # Use a larger port range to avoid colliding with other CI services. if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. steps: @@ -211,7 +210,7 @@ jobs: with: path: ${{ env.CCACHE_DIR }} # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache - key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} + key: ${{ steps.ccache-cache.outputs.cache-primary-key }} windows-native-dll: name: ${{ matrix.job-name }} @@ -253,20 +252,21 @@ jobs: py -3 --version bash --version - - name: Restore vcpkg tools cache - id: vcpkg-tools-cache + - name: Restore vcpkg downloads cache + id: vcpkg-downloads-cache uses: actions/cache/restore@v5 with: - path: ~/AppData/Local/vcpkg/downloads/tools - key: ${{ github.job }}-vcpkg-tools-${{ github.run_id }} - restore-keys: ${{ github.job }}-vcpkg-tools- + path: | + ~/AppData/Local/vcpkg/downloads/* + !~/AppData/Local/vcpkg/downloads/tools + key: ${{ github.job }}-vcpkg-downloads-${{ hashFiles('vcpkg.json') }} - name: Restore vcpkg binary cache uses: actions/cache/restore@v5 id: vcpkg-binary-cache with: path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} + key: ${{ github.job }}-vcpkg-binary-release-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} - name: Generate build system run: | @@ -277,15 +277,18 @@ jobs: if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' with: path: ~/AppData/Local/vcpkg/archives - key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} + key: ${{ steps.vcpkg-binary-cache.outputs.cache-primary-key }} - - name: Save vcpkg tools cache + - name: Save vcpkg downloads cache uses: actions/cache/save@v5 - # Only save cache from one job as they share tools. If the matrix is expanded to jobs with unique tools, this may need amending. - if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.vcpkg-tools-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' + # Only save cache from the 'standard' job, as it includes the necessary downloads for other jobs in the matrix. If the matrix is modified, this may need amending. + if: github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && steps.vcpkg-downloads-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' with: - path: ~/AppData/Local/vcpkg/downloads/tools - key: ${{ github.job }}-vcpkg-tools-${{ github.run_id }} + # Cache the tools once as archives, but not redundantly in extracted form. + path: | + ~/AppData/Local/vcpkg/downloads/* + !~/AppData/Local/vcpkg/downloads/tools + key: ${{ steps.vcpkg-downloads-cache.outputs.cache-primary-key }} - name: Build run: | @@ -322,7 +325,7 @@ jobs: windows-cross: name: 'Windows-cross to x86_64, ${{ matrix.crt }}' needs: [runners, record-frozen-commit] - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-4x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} strategy: @@ -354,18 +357,22 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ needs.runners.outputs.provider }} + provider: ${{ needs.runners.outputs.provider }} - name: CI script run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ needs.runners.outputs.provider }} - name: Upload built executables uses: actions/upload-artifact@v7 @@ -433,7 +440,7 @@ jobs: ci-matrix: name: ${{ matrix.name }} needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && matrix.cirrus-runner || matrix.fallback-runner }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && matrix.warp-runner || matrix.fallback-runner }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: ${{ matrix.timeout-minutes }} @@ -446,93 +453,93 @@ jobs: matrix: include: - name: 'iwyu' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_iwyu.sh' - name: '32 bit ARM' - cirrus-runner: 'ubuntu-24.04-arm' # Cirrus' Arm runners are Apple (with virtual Linux aarch64), which doesn't support 32-bit mode + warp-runner: 'ubuntu-24.04-arm' # Warp's Arm runners don't support 32-bit mode currently fallback-runner: 'ubuntu-24.04-arm' timeout-minutes: 120 file-env: './ci/test/00_setup_env_arm.sh' provider: 'gha' - name: 'ASan + LSan + UBSan + integer' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools + warp-runner: 'warp-ubuntu-2404-x64-8x' # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools fallback-runner: 'ubuntu-24.04' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_asan.sh' - name: 'macOS-cross to arm64' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross.sh' - name: 'macOS-cross to x86_64' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_mac_cross_intel.sh' - name: 'FreeBSD Cross' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_freebsd_cross.sh' - name: 'No wallet' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-sm' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-4x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_nowallet.sh' - name: 'i686, no IPC' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_i686_no_ipc.sh' - name: 'fuzzer,address,undefined,integer' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 240 file-env: './ci/test/00_setup_env_native_fuzz.sh' - name: 'previous releases' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_previous_releases.sh' - name: 'Alpine (musl)' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_alpine_musl.sh' - name: 'tidy' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tidy.sh' - name: 'TSan' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_tsan.sh' - name: 'MSan, fuzz' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-8x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 150 file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh' - name: 'MSan' - cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg' - fallback-runner: 'ubuntu-24.04' + warp-runner: 'warp-ubuntu-latest-x64-16x' + fallback-runner: 'ubuntu-latest' timeout-minutes: 120 file-env: './ci/test/00_setup_env_native_msan.sh' @@ -546,12 +553,14 @@ jobs: - name: Restore caches id: restore-cache - uses: ./.github/actions/restore-caches + uses: ./.github/actions/cache/restore + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ matrix.provider || needs.runners.outputs.provider }} + provider: ${{ matrix.provider || needs.runners.outputs.provider }} - name: Clear unnecessary files if: ${{ needs.runners.outputs.provider == 'gha' && true || false }} # Only needed on GHA runners @@ -572,12 +581,14 @@ jobs: run: ./ci/test_run_all.sh - name: Save caches - uses: ./.github/actions/save-caches + uses: ./.github/actions/cache/save + with: + provider: ${{ matrix.provider || needs.runners.outputs.provider }} lint: name: 'lint' needs: runners - runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }} + runs-on: ${{ needs.runners.outputs.provider == 'warp' && 'warp-ubuntu-latest-x64-2x' || 'ubuntu-latest' }} if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} timeout-minutes: 20 env: @@ -594,7 +605,7 @@ jobs: - name: Configure Docker uses: ./.github/actions/configure-docker with: - cache-provider: ${{ needs.runners.outputs.provider }} + provider: ${{ needs.runners.outputs.provider }} - name: CI script run: | diff --git a/.python-version b/.python-version index 1445aee866ce..c8cfe3959183 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.14 +3.10 diff --git a/CMakeLists.txt b/CMakeLists.txt index eda934cc8205..69ee54611a43 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,6 +80,7 @@ if(CMAKE_VERSION VERSION_LESS 4.2 AND CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND NO set(CMAKE_INSTALL_NAME_TOOL "${CMAKE_COMMAND} -E true") endif() enable_language(CXX) + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -107,84 +108,33 @@ option(BUILD_CLI "Build bitcoin-cli executable." ON) option(BUILD_TESTS "Build test_bitcoin and other unit test executables." ON) option(BUILD_TX "Build bitcoin-tx executable." ${BUILD_TESTS}) option(BUILD_UTIL "Build bitcoin-util executable." ${BUILD_TESTS}) +cmake_dependent_option(BUILD_GUI_TESTS "Build test_bitcoin-qt executable." ON "BUILD_GUI;BUILD_TESTS" OFF) option(BUILD_UTIL_CHAINSTATE "Build experimental bitcoin-chainstate executable." OFF) option(BUILD_KERNEL_LIB "Build experimental bitcoinkernel library." ${BUILD_UTIL_CHAINSTATE}) cmake_dependent_option(BUILD_KERNEL_TEST "Build tests for the experimental bitcoinkernel library." ON "BUILD_KERNEL_LIB" OFF) option(ENABLE_WALLET "Enable wallet." ON) -if(ENABLE_WALLET) - if(VCPKG_TARGET_TRIPLET) - # Use of the `unofficial::` namespace is a vcpkg package manager convention. - find_package(unofficial-sqlite3 CONFIG REQUIRED) - add_library(SQLite3::SQLite3 ALIAS unofficial::sqlite3::sqlite3) - else() - find_package(SQLite3 3.7.17 REQUIRED) - if(NOT TARGET SQLite3::SQLite3) # CMake < 4.3 - add_library(SQLite3::SQLite3 ALIAS SQLite::SQLite3) - endif() - endif() -endif() cmake_dependent_option(BUILD_WALLET_TOOL "Build bitcoin-wallet tool." ${BUILD_TESTS} "ENABLE_WALLET" OFF) -option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting executables." OFF) -option(CMAKE_COMPILE_WARNING_AS_ERROR "Treat compiler warnings as errors." OFF) -option(WITH_CCACHE "Attempt to use ccache for compiling." ON) - +option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) option(WITH_ZMQ "Enable ZMQ notifications." OFF) -if(WITH_ZMQ) - find_package(ZeroMQ 4.0.0 MODULE REQUIRED) -endif() - +cmake_dependent_option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables." ON "NOT WIN32" OFF) +cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF "ENABLE_IPC" OFF) option(WITH_EMBEDDED_ASMAP "Embed default ASMap data." ON) - option(WITH_USDT "Enable tracepoints for Userspace, Statically Defined Tracing." OFF) -if(WITH_USDT) - find_package(USDT MODULE REQUIRED) -endif() - -option(ENABLE_EXTERNAL_SIGNER "Enable external signer support." ON) cmake_dependent_option(WITH_QRENCODE "Enable QR code support." ON "BUILD_GUI" OFF) -if(WITH_QRENCODE) - find_package(QRencode MODULE REQUIRED) - set(USE_QRCODE TRUE) -endif() - cmake_dependent_option(WITH_DBUS "Enable DBus support." ON "NOT CMAKE_SYSTEM_NAME MATCHES \"(Windows|Darwin)\" AND BUILD_GUI" OFF) -cmake_dependent_option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables." ON "NOT WIN32" OFF) -cmake_dependent_option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF "ENABLE_IPC" OFF) -if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS) - find_package(Libmultiprocess REQUIRED COMPONENTS Lib) - find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin - NAMES Libmultiprocess - ) -endif() - -cmake_dependent_option(BUILD_GUI_TESTS "Build test_bitcoin-qt executable." ON "BUILD_GUI;BUILD_TESTS" OFF) -if(BUILD_GUI) - set(qt_components Core Gui Widgets LinguistTools) - if(ENABLE_WALLET) - list(APPEND qt_components Network) - endif() - if(WITH_DBUS) - list(APPEND qt_components DBus) - set(USE_DBUS TRUE) - endif() - if(BUILD_GUI_TESTS) - list(APPEND qt_components Test) - endif() - find_package(Qt 6.2 MODULE REQUIRED - COMPONENTS ${qt_components} - ) - unset(qt_components) -endif() - option(BUILD_BENCH "Build bench_bitcoin executable." OFF) option(BUILD_FUZZ_BINARY "Build fuzz binary." OFF) option(BUILD_FOR_FUZZING "Build for fuzzing. Enabling this will disable all other targets and override BUILD_FUZZ_BINARY." OFF) +option(REDUCE_EXPORTS "Attempt to reduce exported symbols in the resulting executables." OFF) +option(CMAKE_COMPILE_WARNING_AS_ERROR "Treat compiler warnings as errors." OFF) +option(WITH_CCACHE "Attempt to use ccache for compiling." ON) + option(INSTALL_MAN "Install man pages." ON) set(APPEND_CPPFLAGS "" CACHE STRING "Preprocessor flags that are appended to the command line after all other flags added by the build system. This variable is intended for debugging and special builds.") @@ -197,11 +147,13 @@ string(APPEND CMAKE_CXX_COMPILE_OBJECT " ${APPEND_CPPFLAGS} ${APPEND_CXXFLAGS}") string(APPEND CMAKE_CXX_CREATE_SHARED_LIBRARY " ${APPEND_LDFLAGS}") string(APPEND CMAKE_CXX_LINK_EXECUTABLE " ${APPEND_LDFLAGS}") -set(configure_warnings) - -include(CheckLinkerSupportsPIE) -check_linker_supports_pie(configure_warnings) +# Check required C++ toolchain features early. +include(CheckCXXFeatures) +check_cxx_features() +#============================= +# Core interface library +#============================= # The core_interface library aims to encapsulate common build flags. # It is a usage requirement for all targets except for secp256k1, which # gets its flags by other means. @@ -213,6 +165,15 @@ target_link_libraries(core_interface INTERFACE $<$:core_interface_debug> ) +#============================= +# Option interaction +#============================= +# Compute the final state of build options. +# This must be done after all options have been initially set, +# as some options (e.g., BUILD_FOR_FUZZING) may override or adjust others. +# It must also come before any dependency checks to ensure that the set +# of dependencies checked is consistent with the final option values, +# that is, that all required, and only the required, dependencies are tested. if(BUILD_FOR_FUZZING) message(WARNING "BUILD_FOR_FUZZING=ON will disable all other targets and force BUILD_FUZZ_BINARY=ON.") set(BUILD_BITCOIN_BIN OFF) @@ -239,6 +200,81 @@ if(BUILD_FOR_FUZZING) ) endif() +#============================= +# Check and set PIC/PIE +#============================= +set(configure_warnings) +include(CheckLinkerSupportsPIE) +# Set CMAKE_POSITION_INDEPENDENT_CODE early so it applies to all +# subsequent checks, including dependency packages and tool flags. +check_linker_supports_pie(configure_warnings) + +#============================= +# Find dependencies +#============================= +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) +target_link_libraries(core_interface INTERFACE + Threads::Threads +) + +include(AddBoostIfNeeded) +add_boost_if_needed() + +if(BUILD_DAEMON OR BUILD_GUI OR BUILD_CLI OR BUILD_TESTS OR BUILD_BENCH OR BUILD_FUZZ_BINARY) + find_package(Libevent 2.1.8 MODULE REQUIRED) +endif() + +if(ENABLE_WALLET) + if(VCPKG_TARGET_TRIPLET) + # Use of the `unofficial::` namespace is a vcpkg package manager convention. + find_package(unofficial-sqlite3 CONFIG REQUIRED) + add_library(SQLite3::SQLite3 ALIAS unofficial::sqlite3::sqlite3) + else() + find_package(SQLite3 3.7.17 REQUIRED) + if(NOT TARGET SQLite3::SQLite3) # CMake < 4.3 + add_library(SQLite3::SQLite3 ALIAS SQLite::SQLite3) + endif() + endif() +endif() + +if(WITH_ZMQ) + find_package(ZeroMQ 4.0.0 MODULE REQUIRED) +endif() + +if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS) + find_package(Libmultiprocess REQUIRED COMPONENTS Lib) + find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin + NAMES Libmultiprocess + ) +endif() + +if(WITH_USDT) + find_package(USDT MODULE REQUIRED) +endif() + +if(BUILD_GUI) + set(qt_components Core Gui Widgets LinguistTools) + if(ENABLE_WALLET) + list(APPEND qt_components Network) + endif() + if(WITH_DBUS) + list(APPEND qt_components DBus) + set(USE_DBUS TRUE) + endif() + if(BUILD_GUI_TESTS) + list(APPEND qt_components Test) + endif() + find_package(Qt 6.2 MODULE REQUIRED + COMPONENTS ${qt_components} + ) + unset(qt_components) + if(WITH_QRENCODE) + find_package(QRencode MODULE REQUIRED) + set(USE_QRCODE TRUE) + endif() +endif() + include(TryAppendCXXFlags) include(TryAppendLinkerFlag) @@ -349,12 +385,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") endif() endif() -set(THREADS_PREFER_PTHREAD_FLAG ON) -find_package(Threads REQUIRED) -target_link_libraries(core_interface INTERFACE - Threads::Threads -) - # Define sanitize_interface with -fsanitize flags intended to apply to all # libraries and executables. add_library(sanitize_interface INTERFACE) @@ -424,13 +454,6 @@ if(BUILD_FUZZ_BINARY) target_link_libraries(fuzzer_interface INTERFACE ${FUZZ_LIBS}) endif() -include(AddBoostIfNeeded) -add_boost_if_needed() - -if(BUILD_DAEMON OR BUILD_GUI OR BUILD_CLI OR BUILD_TESTS OR BUILD_BENCH OR BUILD_FUZZ_BINARY) - find_package(Libevent 2.1.8 MODULE REQUIRED) -endif() - include(cmake/introspection.cmake) include(cmake/ccache.cmake) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index deb2b40a138b..0505b6b1b7d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,38 +19,18 @@ Getting Started New contributors are very welcome and needed. -Reviewing and testing is highly valued and the most effective way you can contribute -as a new contributor. It also will teach you much more about the code and -process than opening pull requests. Please refer to the [peer review](#peer-review) -section below. +In-depth reviewing and testing are the bottleneck of the project, and are the +most effective way anyone can start to contribute. It will teach you much more +about the code and process than opening pull requests, and may help you uncover +related issues and follow-ups to contribute code for. Please refer to the [peer +review](#peer-review) section below. Before you start contributing, familiarize yourself with the Bitcoin Core build system and tests. Refer to the documentation in the repository on how to build Bitcoin Core and how to run the unit tests, functional tests, and fuzz tests. -There are many open issues of varying difficulty waiting to be fixed. -If you're looking for somewhere to start contributing, check out the -[good first issue](https://github.com/bitcoin/bitcoin/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) -list or changes that are -[up for grabs](https://github.com/bitcoin/bitcoin/issues?utf8=%E2%9C%93&q=label%3A%22Up+for+grabs%22). -Some of them might no longer be applicable. So if you are interested, but -unsure, you might want to leave a comment on the issue first. - You may also participate in the [Bitcoin Core PR Review Club](https://bitcoincore.reviews/). -### Good First Issue Label - -The purpose of the `good first issue` label is to highlight which issues are -suitable for a new contributor without a deep understanding of the codebase. - -However, good first issues can be solved by anyone. If they remain unsolved -for a longer time, a frequent contributor might address them. - -You do not need to request permission to start working on an issue. However, -you are encouraged to leave a comment if you are planning to work on it. This -will help other contributors monitor which issues are actively being addressed -and is also an effective way to request assistance if and when you need it. - Communication Channels ---------------------- @@ -146,7 +126,7 @@ about Git. ### Creating the Pull Request The title of the pull request should be prefixed by the component or area that -the pull request affects. Valid areas as: +the pull request affects. Valid areas are: - `consensus` for changes to consensus critical code - `doc` for changes to the documentation diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 000000000000..539a37d2ff4b --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,2 @@ +set(CTEST_SUBMIT_URL "https://my.cdash.org/submit.php?project=bitcoin-core") +set(CTEST_NIGHTLY_START_TIME "00:00:00 UTC") diff --git a/SECURITY.md b/SECURITY.md index fd4c61d176dc..5a63a762ac2d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,5 +16,6 @@ The following keys may be used to communicate sensitive information to developer | Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | | Michael Ford | E777 299F C265 DD04 7930 70EB 944D 35F9 AC3D B76A | | Ava Chow | 1528 1230 0785 C964 44D3 334D 1756 5732 E08E 5E41 | +| Niklas Gögge | 2CBB F208 E594 BF43 9B5F 276C 7465 CFFF 6793 242E | You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys ""` Ensure that you put quotes around fingerprints containing spaces. diff --git a/ci/README.md b/ci/README.md index 293294a559c5..0b6a34460ca1 100644 --- a/ci/README.md +++ b/ci/README.md @@ -80,20 +80,22 @@ trigger cache-invalidation and rebuilds as necessary. To configure the primary repository, follow these steps: -1. Register with [Cirrus Runners](https://cirrus-runners.app/) and purchase runners. -2. Install the Cirrus Runners GitHub app against the GitHub organization. +1. Register with [WarpBuild](https://www.warpbuild.com/) and purchase runners. +2. Install the WarpBuild GitHub app against the GitHub organization. 3. Enable organisation-level runners to be used in public repositories: 1. `Org settings -> Actions -> Runner Groups -> Default -> Allow public repos` 4. Permit the following actions to run: - 1. cirruslabs/cache/restore@\* - 1. cirruslabs/cache/save@\* - 1. docker/setup-buildx-action@\* + 1. actions/cache/restore@\* + 1. actions/cache/save@\* 1. actions/github-script@\* + 1. docker/setup-buildx-action@\* + 1. warpbuilds/cache/restore@\* + 1. warpbuilds/cache/save@\* ### Forked repositories When used in a fork the CI will run on GitHub's free hosted runners by default. -In this case, due to GitHub's 10GB-per-repo cache size limitations caches will be frequently evicted and missed, but the workflows will run (slowly). +In this case, GitHub's cache size limitations may cause caches to be frequently evicted and missed, but the workflows will run (slowly). -It is also possible to use your own Cirrus Runners in your own fork with an appropriate patch to the `REPO_USE_CIRRUS_RUNNERS` variable in ../.github/workflows/ci.yml -NB that Cirrus Runners only work at an organisation level, therefore in order to use your own Cirrus Runners, *the fork must be within your own organisation*. +It is also possible to use your own WarpBuild Runners in your own fork with an appropriate patch to the `REPO_USE_WARP_RUNNERS` variable in ../.github/workflows/ci.yml +NB that WarpBuild Runners only work at an organisation level, therefore in order to use your own WarpBuild Runners, *the fork must be within your own organisation*. diff --git a/ci/lint/01_install.sh b/ci/lint/01_install.sh index 573dbca5fe56..9c2020a9a15d 100755 --- a/ci/lint/01_install.sh +++ b/ci/lint/01_install.sh @@ -22,29 +22,14 @@ ${CI_RETRY_EXE} apt-get update # - moreutils (used by scripted-diff) ${CI_RETRY_EXE} apt-get install -y cargo curl xz-utils git gpg moreutils -PYTHON_PATH="/python_build" -if [ ! -d "${PYTHON_PATH}/bin" ]; then - ( - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/pyenv/pyenv.git - cd pyenv/plugins/python-build || exit 1 - ./install.sh - ) - # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment - ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ - clang - env CC=clang python-build "$(cat "/.python-version")" "${PYTHON_PATH}" -fi -export PATH="${PYTHON_PATH}/bin:${PATH}" +# Install Python and create venv using uv (reads version from .python-version) +uv venv /python_env + +export PATH="/python_env/bin:${PATH}" command -v python3 python3 --version -${CI_RETRY_EXE} pip3 install \ - lief==0.17.5 \ - mypy==1.19.1 \ - pyzmq==27.1.0 \ - ruff==0.15.5 +uv pip install --python /python_env --requirements /ci/lint/requirements.txt SHELLCHECK_VERSION=v0.11.0 curl --fail -L "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.$(uname --machine).tar.xz" | \ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 1b36fada9fc6..a0f2dc88bc47 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -12,7 +12,7 @@ set -o errexit -o pipefail -o xtrace # of the mounted bitcoin src dir. git config --global --add safe.directory /bitcoin -export PATH="/python_build/bin:${PATH}" +export PATH="/python_env/bin:${PATH}" if [ -n "${LINT_CI_IS_PR}" ]; then export COMMIT_RANGE="HEAD~..HEAD" diff --git a/ci/lint/requirements.txt b/ci/lint/requirements.txt new file mode 100644 index 000000000000..e8abf041dc76 --- /dev/null +++ b/ci/lint/requirements.txt @@ -0,0 +1,3 @@ +lief==0.17.5 +mypy==1.19.1 +pyzmq==27.1.0 diff --git a/ci/lint_imagefile b/ci/lint_imagefile index 77e9688c6506..a6c49a7c8619 100644 --- a/ci/lint_imagefile +++ b/ci/lint_imagefile @@ -6,8 +6,15 @@ FROM mirror.gcr.io/ubuntu:24.04 +# Pin uv and ruff to minor version to avoid breaking changes +# https://docs.astral.sh/uv/reference/policies/versioning/ +# https://docs.astral.sh/ruff/versioning/ +COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /uvx /bin/ +COPY --from=ghcr.io/astral-sh/ruff:0.15 /ruff /bin/ + COPY ./ci/retry/retry /ci_retry COPY ./.python-version /.python-version +COPY ./ci/lint/requirements.txt /ci/lint/requirements.txt COPY ./ci/lint/01_install.sh /install.sh RUN /install.sh && \ diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 8d9a16ab7841..4ca660a5d69e 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -set -o errexit -o pipefail -o xtrace +set -o errexit -o nounset -o pipefail -o xtrace # The source root dir, usually from git, usually read-only. # The ci system copies this folder. @@ -20,15 +20,14 @@ export BASE_ROOT_DIR="${BASE_ROOT_DIR:-/ci_container_base}" # This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} # A folder for the ci system to put temporary files (build result, datadirs for tests, ...) +# The name contains a space and a non-ASCII symbol to confirm the build and +# tests handle word-splitting and UTF8 correctly. # This folder only exists on the ci guest. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch_ ₿🧪_} echo "Setting specific values in env" -if [ -n "${FILE_ENV}" ]; then - set -o errexit; - # shellcheck disable=SC1090 - source "${FILE_ENV}" -fi +# shellcheck disable=SC1090 +source "${FILE_ENV}" echo "Fallback to default values in env (if not yet set)" # The number of parallel jobs to pass down to make and test_runner.py diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index f4ad97d2cc2b..71b519dab189 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8 export HOST=arm-linux-gnueabihf export DPKG_ADD_ARCH="armhf" -export PACKAGES="python3-zmq g++-arm-linux-gnueabihf libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PACKAGES="python3-pip python3-zmq g++-arm-linux-gnueabihf libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf" +export PIP_PACKAGES="--break-system-packages pycapnp" export CONTAINER_NAME=ci_arm_linux export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" # Check that https://packages.debian.org/trixie/g++-arm-linux-gnueabihf (version 14.x, similar to guix) can cross-compile export CI_IMAGE_PLATFORM="linux/arm64" diff --git a/ci/test/00_setup_env_i686_no_ipc.sh b/ci/test/00_setup_env_i686_no_ipc.sh index dca0486fb602..93d5ab744604 100755 --- a/ci/test/00_setup_env_i686_no_ipc.sh +++ b/ci/test/00_setup_env_i686_no_ipc.sh @@ -10,11 +10,11 @@ export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_no_multiprocess export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:trixie" export CI_IMAGE_PLATFORM="linux/amd64" +export CI_CONTAINER_CAP="--security-opt seccomp=unconfined" export PACKAGES="llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 NO_IPC=1" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 -export TEST_RUNNER_EXTRA="--v2transport --usecli" export BITCOIN_CONFIG="\ --preset=dev-mode \ -DENABLE_IPC=OFF \ diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index a9b4d5c8c2e5..d59c5ae65c55 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME="ci_mac_native" # macos does not use a container, but the env var is needed for logging -export PIP_PACKAGES="--break-system-packages pycapnp zmq" +export PIP_PACKAGES="--break-system-packages pycapnp pyzmq" export GOAL="install deploy" export CMAKE_GENERATOR="Ninja" export CI_OS_NAME="macos" diff --git a/ci/test/00_setup_env_native_alpine_musl.sh b/ci/test/00_setup_env_native_alpine_musl.sh index e42cb4a3f1e8..3bad08d68538 100755 --- a/ci/test/00_setup_env_native_alpine_musl.sh +++ b/ci/test/00_setup_env_native_alpine_musl.sh @@ -17,4 +17,5 @@ export BITCOIN_CONFIG="\ -DREDUCE_EXPORTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ " +export TEST_RUNNER_EXTRA="--v2transport --usecli --extended --exclude feature_dbcrash" # Run extended tests under --usecli and --v2transport, but exclude the very slow dbcrash export BITCOIN_CMD="bitcoin -m" # Used in functional tests diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index c6923896c562..82d9044a8d70 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -10,8 +10,7 @@ export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04" export APT_LLVM_V="22" LIBCXX_DIR="/cxx_build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" -# -lstdc++ to resolve link issues due to upstream packaging -LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument -lstdc++" +LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 4a17fb496d60..4a10bf96eabc 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-pip" -export PIP_PACKAGES="--break-system-packages pycapnp" +export PIP_PACKAGES="--break-system-packages pycapnp pyzmq" export DEP_OPTS="DEBUG=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export CI_LIMIT_STACK_SIZE=1 diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 5a6e06811fa0..8b10ba10baac 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -62,10 +62,13 @@ if [ -n "$PIP_PACKAGES" ]; then fi if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then - ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-22.1.0" /llvm-project + ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-22.1.7" /llvm-project +# LLVM is configured with LIBCXXABI_USE_LLVM_UNWINDER=OFF, +# because libunwind doesn't handle exceptions under MSAN. +# https://github.com/llvm/llvm-project/issues/84348 cmake -G Ninja -B /cxx_build/ \ - -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \ + -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_USE_SANITIZER="${USE_INSTRUMENTED_LIBCPP}" \ -DCMAKE_C_COMPILER=clang \ diff --git a/ci/test/01_iwyu.patch b/ci/test/01_iwyu.patch index d86a02dc7669..794bb74f5875 100644 --- a/ci/test/01_iwyu.patch +++ b/ci/test/01_iwyu.patch @@ -15,7 +15,7 @@ See: https://en.cppreference.com/w/cpp/preprocessor/include.html. Prefer C++ headers over C counterparts. -See: https://github.com/include-what-you-use/include-what-you-use/blob/clang_21/iwyu_include_picker.cc#L587-L629. +See: https://github.com/include-what-you-use/include-what-you-use/blob/clang_22/iwyu_include_picker.cc#L645-L687. --- a/iwyu_include_picker.cc +++ b/iwyu_include_picker.cc diff --git a/ci/test/02_run_container.py b/ci/test/02_run_container.py index abaa535553db..05887000a2ba 100755 --- a/ci/test/02_run_container.py +++ b/ci/test/02_run_container.py @@ -115,6 +115,7 @@ def main(): CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}" run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False) + run(["docker", "network", "create", "--subnet", "1.1.1.0/24", "ci-ip4net"], check=False) if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"): print("Restart docker before run to stop and clear all containers started with --rm") @@ -144,6 +145,7 @@ def main(): f"--env-file={env_file}", f"--name={os.environ['CONTAINER_NAME']}", "--network=ci-ip6net", + "--ip6=1111:1111::5", # Used by some of the tests, don't change it just here (keep them in sync). f"--platform={os.environ['CI_IMAGE_PLATFORM']}", os.environ["CONTAINER_NAME"], ] @@ -154,6 +156,8 @@ def main(): text=True, ).stdout.strip() + run(["docker", "network", "connect", "--ip=1.1.1.5", "ci-ip4net", container_id]) # The IP address is used by some of the tests, don't change it just here (keep them in sync). + def ci_exec(cmd_inner, **kwargs): if os.getenv("DANGER_RUN_CI_ON_HOST"): prefix = [] diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 4e4d975778dd..980583d00b56 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -set -o errexit -o xtrace +set -o errexit -o xtrace -o pipefail if [ "${DANGER_RUN_CI_ON_HOST}" != "1" ]; then echo "This script will make unsafe local and global modifications, so it can only be run inside a container and requires DANGER_RUN_CI_ON_HOST=1" @@ -109,7 +109,7 @@ ccache --zero-stats # Folder where the build is done. BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST} -BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR -Werror=dev" +BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL '-DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR' -Werror=dev" if [[ "${RUN_IWYU}" == true || "${RUN_TIDY}" == true ]]; then BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" @@ -136,10 +136,30 @@ cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target $GOAL || ( ) ccache --version | head -n 1 && ccache --show-stats --verbose -hit_rate=$(ccache --show-stats | grep "Hits:" | head -1 | sed 's/.*(\(.*\)%).*/\1/') -if [ "${hit_rate%.*}" -lt 75 ]; then - echo "::notice title=low ccache hitrate::Ccache hit-rate in $CONTAINER_NAME was $hit_rate%" -fi +ccache --print-stats | python3 -c ' +import os +import sys + +for line in sys.stdin: + key, value = line.split("\t", 1) + # "primary storage" fallback only needed for ccache version 4.5.1 + if key in ("local_storage_hit", "primary_storage_hit"): + hits = int(value) + elif key in ("local_storage_miss", "primary_storage_miss"): + miss = int(value) + +calls = hits + miss +# codegen has no calls, so skip that here +if calls: + rate = (hits / calls * 100) + print(f"{rate:.2f}") + if rate < 75: + container = os.environ["CONTAINER_NAME"] + print( + "::notice title=low ccache hitrate::" + f"Ccache hit-rate in {container} was {rate:.2f}%" + ) +' du -sh "${DEPENDS_DIR}"/*/ du -sh "${PREVIOUS_RELEASES_DIR}" @@ -209,7 +229,7 @@ fi if [[ "${RUN_IWYU}" == true ]]; then # TODO: Consider enforcing IWYU across the entire codebase. - FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|univalue/(lib|test)|util|zmq)/.*|common/license_info|node/blockstorage|node/utxo_snapshot|clientversion|core_io|signet)\\.cpp)" + FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|script|univalue/(lib|test)|util|zmq)/.*|bench/(block_assemble|connectblock)|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json" @@ -217,12 +237,14 @@ if [[ "${RUN_IWYU}" == true ]]; then run_iwyu() { mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json" - python3 "/include-what-you-use/iwyu_tool.py" \ + { + python3 "/include-what-you-use/iwyu_tool.py" \ -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \ -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \ -Xiwyu --max_line_length=160 \ -Xiwyu --check_also="*/primitives/*.h" \ - 2>&1 | tee /tmp/iwyu_ci.out + 2>&1 || true + } | tee /tmp/iwyu_ci.out python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git diff -U1 | ./contrib/devtools/clang-format-diff.py -binary="clang-format-${IWYU_LLVM_V}" -p1 -i -v } diff --git a/cmake/leveldb.cmake b/cmake/leveldb.cmake index cff8c61f96ca..b9246cf81030 100644 --- a/cmake/leveldb.cmake +++ b/cmake/leveldb.cmake @@ -56,7 +56,6 @@ target_compile_definitions(leveldb HAVE_FDATASYNC=$ HAVE_FULLFSYNC=$ HAVE_O_CLOEXEC=$ - FALLTHROUGH_INTENDED=[[fallthrough]] $<$>:LEVELDB_PLATFORM_POSIX> $<$:LEVELDB_PLATFORM_WINDOWS> $<$:_UNICODE;UNICODE> @@ -84,9 +83,6 @@ if(MSVC) _CRT_NONSTDC_NO_WARNINGS ) else() - try_append_cxx_flags("-Wconditional-uninitialized" TARGET nowarn_leveldb_interface SKIP_LINK - IF_CHECK_PASSED "-Wno-conditional-uninitialized" - ) try_append_cxx_flags("-Wcovered-switch-default" TARGET nowarn_leveldb_interface SKIP_LINK IF_CHECK_PASSED "-Wno-covered-switch-default" ) diff --git a/cmake/module/AddBoostIfNeeded.cmake b/cmake/module/AddBoostIfNeeded.cmake index 80a6d2e89117..658e6d07c9ad 100644 --- a/cmake/module/AddBoostIfNeeded.cmake +++ b/cmake/module/AddBoostIfNeeded.cmake @@ -31,17 +31,6 @@ function(add_boost_if_needed) find_package(Boost 1.74.0 REQUIRED CONFIG) mark_as_advanced(Boost_INCLUDE_DIR boost_headers_DIR) - # Workaround for a bug in NetBSD pkgsrc. - # See https://gnats.netbsd.org/59856. - if(CMAKE_SYSTEM_NAME STREQUAL "NetBSD") - get_filename_component(_boost_include_dir "${boost_headers_DIR}/../../../include/" ABSOLUTE) - if(_boost_include_dir MATCHES "^/usr/pkg/") - set_target_properties(Boost::headers PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${_boost_include_dir} - ) - endif() - unset(_boost_include_dir) - endif() set_target_properties(Boost::headers PROPERTIES IMPORTED_GLOBAL TRUE) target_compile_definitions(Boost::headers INTERFACE # We don't use multi_index serialization. diff --git a/cmake/module/CheckCXXFeatures.cmake b/cmake/module/CheckCXXFeatures.cmake new file mode 100644 index 000000000000..fde630e30d09 --- /dev/null +++ b/cmake/module/CheckCXXFeatures.cmake @@ -0,0 +1,43 @@ +# Copyright (c) 2026-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +include_guard(GLOBAL) + +# Checks for C++ features required to compile Bitcoin Core. + +include(CheckCXXSourceCompiles) + +function(check_cxx_features) + set(CMAKE_REQUIRED_QUIET TRUE) + + message(STATUS "Checking for required C++ features") + + # Checks for Class Template Argument Deduction for aggregate types - used in src/util/overloaded.h + check_cxx_source_compiles(" + #include + + template struct Overloaded : Ts... { using Ts::operator()...; }; + + int main() { + std::variant v = 42; + return std::visit(Overloaded{ + [](int) { return 0; }, + [](double) { return 1; } + }, v); + } + " HAVE_CTAD_FOR_AGGREGATES) + + if(NOT HAVE_CTAD_FOR_AGGREGATES) + message(FATAL_ERROR + "Compiler lacks Class Template Argument Deduction (CTAD) for aggregates.\n" + "This C++ feature is required for src/util/overloaded.h.\n" + "You are probably using an old compiler version\n" + "The recommended compiler versions can be checked in\n" + "doc/dependencies.md#compiler.\n" + ) + endif() + + message(STATUS "Checking for required C++ features - done") + +endfunction() diff --git a/cmake/script/GenerateBuildInfo.cmake b/cmake/script/GenerateBuildInfo.cmake index d3ee2eb06210..62773a3a5439 100644 --- a/cmake/script/GenerateBuildInfo.cmake +++ b/cmake/script/GenerateBuildInfo.cmake @@ -5,7 +5,7 @@ macro(fatal_error) message(FATAL_ERROR "\n" "Usage:\n" - " cmake -D BUILD_INFO_HEADER_PATH= [-D SOURCE_DIR=] -P ${CMAKE_CURRENT_LIST_FILE}\n" + " cmake -D BUILD_INFO_HEADER_PATH= -D GIT_EXECUTABLE= [-D SOURCE_DIR=] -P ${CMAKE_CURRENT_LIST_FILE}\n" "All specified paths must be absolute ones.\n" ) endmacro() @@ -28,72 +28,69 @@ else() set(WORKING_DIR ${CMAKE_CURRENT_SOURCE_DIR}) endif() -set(GIT_TAG) -set(GIT_COMMIT) -if(NOT "$ENV{BITCOIN_GENBUILD_NO_GIT}" STREQUAL "1") - find_package(Git QUIET) - if(Git_FOUND) +set(GIT_TAG "") +set(GIT_COMMIT "") +if(GIT_EXECUTABLE) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --is-inside-work-tree + WORKING_DIRECTORY ${WORKING_DIR} + OUTPUT_VARIABLE IS_INSIDE_WORK_TREE + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + if(IS_INSIDE_WORK_TREE) + # Clean 'dirty' status of touched files that haven't been modified. + execute_process( + COMMAND ${GIT_EXECUTABLE} diff + WORKING_DIRECTORY ${WORKING_DIR} + OUTPUT_QUIET + ERROR_QUIET + ) + execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --is-inside-work-tree + COMMAND ${GIT_EXECUTABLE} describe --abbrev=0 WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_VARIABLE IS_INSIDE_WORK_TREE + OUTPUT_VARIABLE MOST_RECENT_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) - if(IS_INSIDE_WORK_TREE) - # Clean 'dirty' status of touched files that haven't been modified. - execute_process( - COMMAND ${GIT_EXECUTABLE} diff - WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_QUIET - ERROR_QUIET - ) - execute_process( - COMMAND ${GIT_EXECUTABLE} describe --abbrev=0 - WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_VARIABLE MOST_RECENT_TAG - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-list -1 ${MOST_RECENT_TAG} + WORKING_DIRECTORY ${WORKING_DIR} + OUTPUT_VARIABLE MOST_RECENT_TAG_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) - execute_process( - COMMAND ${GIT_EXECUTABLE} rev-list -1 ${MOST_RECENT_TAG} - WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_VARIABLE MOST_RECENT_TAG_COMMIT - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse HEAD + WORKING_DIRECTORY ${WORKING_DIR} + OUTPUT_VARIABLE HEAD_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + execute_process( + COMMAND ${GIT_EXECUTABLE} diff-index --quiet HEAD -- + WORKING_DIRECTORY ${WORKING_DIR} + RESULT_VARIABLE IS_DIRTY + ) + + if(HEAD_COMMIT STREQUAL MOST_RECENT_TAG_COMMIT AND NOT IS_DIRTY) + # If latest commit is tagged and not dirty, then use the tag name. + set(GIT_TAG ${MOST_RECENT_TAG}) + else() + # Otherwise, generate suffix from git, i.e. string like "0e0a5173fae3-dirty". execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse HEAD + COMMAND ${GIT_EXECUTABLE} rev-parse --short=12 HEAD WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_VARIABLE HEAD_COMMIT + OUTPUT_VARIABLE GIT_COMMIT OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET ) - - execute_process( - COMMAND ${GIT_EXECUTABLE} diff-index --quiet HEAD -- - WORKING_DIRECTORY ${WORKING_DIR} - RESULT_VARIABLE IS_DIRTY - ) - - if(HEAD_COMMIT STREQUAL MOST_RECENT_TAG_COMMIT AND NOT IS_DIRTY) - # If latest commit is tagged and not dirty, then use the tag name. - set(GIT_TAG ${MOST_RECENT_TAG}) - else() - # Otherwise, generate suffix from git, i.e. string like "0e0a5173fae3-dirty". - execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --short=12 HEAD - WORKING_DIRECTORY ${WORKING_DIR} - OUTPUT_VARIABLE GIT_COMMIT - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - if(IS_DIRTY) - string(APPEND GIT_COMMIT "-dirty") - endif() + if(IS_DIRTY) + string(APPEND GIT_COMMIT "-dirty") endif() endif() endif() diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 941879441bb1..c5ec730917b7 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -50,8 +50,13 @@ versions = [] for relpath in BINARIES: abspath = os.path.join(builddir, relpath) + # Prevent QT from emitting a localized version string for --version and --help + is_qt = relpath == "bin/bitcoin-qt" try: - r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, text=True) + cmd_args = ["--version"] + if is_qt: + cmd_args.append("--lang=en") + r = subprocess.run([abspath, *cmd_args], stdout=subprocess.PIPE, check=True, text=True) except IOError: if(args.skip_missing_binaries): print(f'{abspath} not found or not an executable. Skipping...', file=sys.stderr) @@ -69,13 +74,13 @@ copyright = r.stdout.split('\n')[1:] assert copyright[0].startswith('Copyright (C)') - versions.append((abspath, verstr, copyright)) + versions.append((abspath, verstr, copyright, is_qt)) if not versions: print(f'No binaries found in {builddir}. Please ensure the binaries are present in {builddir}, or set another build path using the BUILDDIR env variable.') sys.exit(1) -if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): +if any(verstr.endswith('-dirty') for (_, verstr, _, _) in versions): print("WARNING: Binaries were built from a dirty tree.") print('man pages generated from dirty binaries should NOT be committed.') print('To properly generate man pages, please commit your changes (or discard them), rebuild, then run this script again.') @@ -93,7 +98,16 @@ footer.flush() # Call the binaries through help2man to produce a manual page for each of them. - for (abspath, verstr, _) in versions: + for (abspath, verstr, _, is_qt) in versions: outname = os.path.join(mandir, os.path.basename(abspath) + '.1') + help_option = '--help-option=--help' + (' --lang=en' if is_qt else '') print(f'Generating {outname}…') - subprocess.run([help2man, '-N', '--version-string=' + verstr, '--include=' + footer.name, '-o', outname, abspath], check=True) + subprocess.run([ + help2man, + '-N', + '--version-string=' + verstr, + '--include=' + footer.name, + '-o', outname, + help_option, + abspath, + ], check=True) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index b0ef28dc3f92..3d70731cbdb4 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -139,14 +139,6 @@ fi ## Attest ## ############## -# Usage: out_name $outdir -# -# HOST: The output directory being attested -# -out_name() { - basename "$(dirname "$1")" -} - shasum_already_exists() { cat < "${VAR_BASE}/precious_dirs" @@ -305,22 +286,6 @@ for host in $HOSTS; do make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} done -# Usage: outdir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - ######### # BUILD # @@ -356,7 +321,6 @@ for host in $HOSTS; do # for the particular $HOST we're building for export HOST="$host" - # shellcheck disable=SC2030 cat << EOF INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} @@ -364,9 +328,9 @@ INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...bind-mounted in container to: '$(distsrc_for_host "$HOST" "" /distsrc-base)' ...outputting in: '$(outdir_for_host "$HOST")' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ...bind-mounted in container to: '$(outdir_for_host "$HOST" "" /outdir-base)' ADDITIONAL FLAGS (if set) ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} @@ -440,8 +404,8 @@ EOF # Please read the README.md in the same directory as this file for # more information. # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + # shellcheck disable=SC2086 + time-machine shell --manifest="${PWD}/contrib/guix/manifest_build.scm" \ --container \ --writable-root \ --pure \ @@ -468,8 +432,8 @@ EOF ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ + DISTSRC="$(distsrc_for_host "$HOST" "" /distsrc-base)" \ + OUTDIR="$(outdir_for_host "$HOST" "" /outdir-base)" \ DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" ) diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 39291dfe9ceb..8cc6993b824d 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -99,18 +99,10 @@ fi # Default to building for all supported HOSTs (overridable by environment) export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}" -# Usage: distsrc_for_host HOST -# -# HOST: The current platform triple we're building for -# -distsrc_for_host() { - echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}-codesigned" -} - # Accumulate a list of build directories that already exist... hosts_distsrc_exists="" for host in $HOSTS; do - if [ -e "$(distsrc_for_host "$host")" ]; then + if [ -e "$(distsrc_for_host "$host" codesigned)" ]; then hosts_distsrc_exists+=" ${host}" fi done @@ -134,7 +126,7 @@ packages cache, the garbage collector roots for Guix environments, and the output directory. EOF for host in $hosts_distsrc_exists; do - echo " ${host} '$(distsrc_for_host "$host")'" + echo " ${host} '$(distsrc_for_host "$host" codesigned)'" done exit 1 else @@ -146,22 +138,18 @@ fi # Codesigning tarballs SHOULD exist ################ -# Usage: outdir_for_host HOST SUFFIX +# Usage: codesigning_tarball_for_host HOST [BASE] # # HOST: The current platform triple we're building for +# BASE: Optional. If provided, replaces ${OUTDIR_BASE} # -outdir_for_host() { - echo "${OUTDIR_BASE}/${1}${2:+-${2}}" -} - - codesigning_tarball_for_host() { case "$1" in *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win64-codesigning.tar.gz" + echo "$(outdir_for_host "$1" "" "$2")/${DISTNAME}-win64-codesigning.tar.gz" ;; *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-codesigning.tar.gz" + echo "$(outdir_for_host "$1" "" "$2")/${DISTNAME}-${1}-codesigning.tar.gz" ;; *) exit 1 @@ -232,14 +220,6 @@ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --f OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" mkdir -p "$OUTDIR_BASE" -# Usage: profiledir_for_host HOST SUFFIX -# -# HOST: The current platform triple we're building for -# -profiledir_for_host() { - echo "${PROFILES_BASE}/${1}${2:+-${2}}" -} - ######### # BUILD # ######### @@ -274,16 +254,15 @@ for host in $HOSTS; do # for the particular $HOST we're building for export HOST="$host" - # shellcheck disable=SC2030 cat << EOF INFO: Codesigning ${VERSION:?not set} for platform triple ${HOST:?not set}: ...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} ...from worktree directory: '${PWD}' ...bind-mounted in container to: '/bitcoin' - ...in build directory: '$(distsrc_for_host "$HOST")' - ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' + ...in build directory: '$(distsrc_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(distsrc_for_host "$HOST" codesigned /distsrc-base)' ...outputting in: '$(outdir_for_host "$HOST" codesigned)' - ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)' + ...bind-mounted in container to: '$(outdir_for_host "$HOST" codesigned /outdir-base)' ...using detached signatures in: '${DETACHED_SIGS_REPO:?not set}' ...bind-mounted in container to: '/detached-sigs' EOF @@ -340,8 +319,8 @@ EOF # Please read the README.md in the same directory as this file for # more information. # - # shellcheck disable=SC2086,SC2031 - time-machine shell --manifest="${PWD}/contrib/guix/manifest.scm" \ + # shellcheck disable=SC2086 + time-machine shell --manifest="${PWD}/contrib/guix/manifest_codesign.scm" \ --container \ --writable-root \ --pure \ @@ -364,11 +343,11 @@ EOF JOBS="$JOBS" \ SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ ${V:+V=1} \ - DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ - OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST" codesigned)" \ + DISTSRC="$(distsrc_for_host "$HOST" codesigned /distsrc-base)" \ + OUTDIR="$(outdir_for_host "$HOST" codesigned /outdir-base)" \ DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ DETACHED_SIGS_REPO=/detached-sigs \ - CODESIGNING_TARBALL="$(OUTDIR_BASE=/outdir-base && codesigning_tarball_for_host "$HOST")" \ + CODESIGNING_TARBALL="$(codesigning_tarball_for_host "$HOST" /outdir-base)" \ bash -c "cd /bitcoin && bash contrib/guix/libexec/codesign.sh" ) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 51354273efae..d77bc5aacd16 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -5,73 +5,13 @@ export LC_ALL=C set -e -o pipefail -# Environment variables for determinism -export TAR_OPTIONS="--no-same-owner --owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" -export TZ=UTC - -# Although Guix _does_ set umask when building its own packages (in our case, -# this is all packages in manifest.scm), it does not set it for `guix -# shell`. It does make sense for at least `guix shell --container` -# to set umask, so if that change gets merged upstream and we bump the -# time-machine to a commit which includes the aforementioned change, we can -# remove this line. -# -# This line should be placed before any commands which creates files. -umask 0022 - -if [ -n "$V" ]; then - # Print both unexpanded (-v) and expanded (-x) forms of commands as they are - # read from this file. - set -vx - # Set VERBOSE for CMake-based builds - export VERBOSE="$V" -fi - -# Check that required environment variables are set -cat << EOF -Required environment variables as seen inside the container: - DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} - DISTNAME: ${DISTNAME:?not set} - HOST: ${HOST:?not set} - SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} - JOBS: ${JOBS:?not set} - DISTSRC: ${DISTSRC:?not set} - OUTDIR: ${OUTDIR:?not set} -EOF - -ACTUAL_OUTDIR="${OUTDIR}" -OUTDIR="${DISTSRC}/output" - -##################### -# Environment Setup # -##################### - -# The depends folder also serves as a base-prefix for depends packages for -# $HOSTs after successfully building. -BASEPREFIX="${PWD}/depends" - -# Given a package name and an output name, return the path of that output in our -# current guix environment -store_path() { - grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ - | head --lines=1 \ - | sed --expression='s|\x29*$||' \ - --expression='s|^[[:space:]]*"||' \ - --expression='s|"[[:space:]]*$||' -} - +# shellcheck source=setup.sh +source "$(dirname "${BASH_SOURCE[0]}")/setup.sh" # Set environment variables to point the NATIVE toolchain to the right # includes/libs NATIVE_GCC="$(store_path gcc-toolchain)" -unset LIBRARY_PATH -unset CPATH -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset OBJC_INCLUDE_PATH -unset OBJCPLUS_INCLUDE_PATH - # Set native toolchain build_CC="${NATIVE_GCC}/bin/gcc -isystem ${NATIVE_GCC}/include" build_CXX="${NATIVE_GCC}/bin/g++ -isystem ${NATIVE_GCC}/include/c++ -isystem ${NATIVE_GCC}/include" @@ -134,15 +74,6 @@ for p in "${PATHS[@]}"; do fi done -# Disable Guix ld auto-rpath behavior -export GUIX_LD_WRAPPER_DISABLE_RPATH=yes - -# Make /usr/bin if it doesn't exist -[ -e /usr/bin ] || mkdir -p /usr/bin - -# Symlink env to a conventional path -[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env - # Determine the correct value for -Wl,--dynamic-linker for the current $HOST case "$HOST" in *linux*) @@ -186,20 +117,6 @@ case "$HOST" in ;; esac -########################### -# Source Tarball Building # -########################### - -GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" - -# Create the source tarball if not already there -if [ ! -e "$GIT_ARCHIVE" ]; then - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD -fi - -mkdir -p "$OUTDIR" - ########################### # Binary Tarball Building # ########################### diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 166675e8bf09..238527678979 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,7 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash - -# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ @@ -80,6 +77,34 @@ time-machine() { -- "$@" } +# Usage: distsrc_for_host HOST [SUFFIX] [BASE] +# +# HOST: The current platform triple we're building for +# SUFFIX: Optional. If provided, appended to the directory name as "-SUFFIX" +# BASE: Optional. If provided, replaces ${DISTSRC_BASE} +# +distsrc_for_host() { + echo "${3:-${DISTSRC_BASE}}/distsrc-${VERSION}-${1}${2:+-${2}}" +} + +# Usage: outdir_for_host HOST [SUFFIX] [BASE] +# +# HOST: The current platform triple we're building for +# SUFFIX: Optional. If provided, appended to the directory name as "-SUFFIX" +# BASE: Optional. If provided, replaces ${OUTDIR_BASE} +# +outdir_for_host() { + echo "${3:-${OUTDIR_BASE}}/${1}${2:+-${2}}" +} + +# Usage: profiledir_for_host HOST [SUFFIX] +# +# HOST: The current platform triple we're building for +# SUFFIX: Optional. If provided, appended to the directory name as "-SUFFIX" +# +profiledir_for_host() { + echo "${PROFILES_BASE}/${1}${2:+-${2}}" +} ################ # Set common variables diff --git a/contrib/guix/libexec/setup.sh b/contrib/guix/libexec/setup.sh new file mode 100755 index 000000000000..617a7327f7bf --- /dev/null +++ b/contrib/guix/libexec/setup.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Copyright (c) The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit. +export LC_ALL=C +set -e -o pipefail + +# Environment variables for determinism +export TAR_OPTIONS="--no-same-owner --owner=0 --group=0 --numeric-owner --mtime='@${SOURCE_DATE_EPOCH}' --sort=name" +export TZ=UTC + +# Although Guix _does_ set umask when building its own packages (in our case, +# this is all packages in manifest.scm), it does not set it for `guix +# shell`. It does make sense for at least `guix shell --container` +# to set umask, so if that change gets merged upstream and we bump the +# time-machine to a commit which includes the aforementioned change, we can +# remove this line. +# +# This line should be placed before any commands which creates files. +umask 0022 + +if [ -n "$V" ]; then + # Print both unexpanded (-v) and expanded (-x) forms of commands as they are + # read from this file. + set -vx + # Set VERBOSE for CMake-based builds + export VERBOSE="$V" +fi + +# Check that required environment variables are set +cat << EOF +Required environment variables as seen inside the container: + DIST_ARCHIVE_BASE: ${DIST_ARCHIVE_BASE:?not set} + DISTNAME: ${DISTNAME:?not set} + HOST: ${HOST:?not set} + SOURCE_DATE_EPOCH: ${SOURCE_DATE_EPOCH:?not set} + JOBS: ${JOBS:?not set} + DISTSRC: ${DISTSRC:?not set} + OUTDIR: ${OUTDIR:?not set} +EOF + +export ACTUAL_OUTDIR="${OUTDIR}" +export OUTDIR="${DISTSRC}/output" + +##################### +# Environment Setup # +##################### + +# The depends folder also serves as a base-prefix for depends packages for +# $HOSTs after successfully building. +export BASEPREFIX="${PWD}/depends" + +# Given a package name and an output name, return the path of that output in our +# current guix environment +store_path() { + grep --extended-regexp "/[^-]{32}-${1}-[^-]+${2:+-${2}}" "${GUIX_ENVIRONMENT}/manifest" \ + | head --lines=1 \ + | sed --expression='s|\x29*$||' \ + --expression='s|^[[:space:]]*"||' \ + --expression='s|"[[:space:]]*$||' +} + +# Disable Guix ld auto-rpath behavior +export GUIX_LD_WRAPPER_DISABLE_RPATH=yes + +# Make /usr/bin if it doesn't exist +[ -e /usr/bin ] || mkdir -p /usr/bin + +# Symlink env to a conventional path +[ -e /usr/bin/env ] || ln -s --no-dereference "$(command -v env)" /usr/bin/env + +########################### +# Source Tarball Building # +########################### + +GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD +fi + +mkdir -p "$OUTDIR" + +unset LIBRARY_PATH +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset OBJC_INCLUDE_PATH +unset OBJCPLUS_INCLUDE_PATH diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest_build.scm similarity index 64% rename from contrib/guix/manifest.scm rename to contrib/guix/manifest_build.scm index 0771524bff30..576021e60d7f 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest_build.scm @@ -3,9 +3,8 @@ (gnu packages bison) ((gnu packages cmake) #:select (cmake-minimal)) (gnu packages commencement) - (gnu packages compression) + ((gnu packages compression) #:select (gzip xz zip)) (gnu packages cross-base) - ((gnu packages crypto) #:select (osslsigncode)) (gnu packages gawk) (gnu packages gcc) ((gnu packages installers) #:select (nsis-x86_64)) @@ -15,14 +14,10 @@ (gnu packages ninja) (gnu packages pkg-config) ((gnu packages python) #:select (python-minimal)) - ((gnu packages python-build) #:select (python-poetry-core)) - ((gnu packages python-crypto) #:select (python-asn1crypto python-oscrypto)) ((gnu packages python-xyz) #:select (python-lief)) - ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) - (guix build-system python) - (guix build-system pyproject) (guix build-system trivial) + (guix download) (guix gexp) (guix git-download) ((guix licenses) #:prefix license:) @@ -38,13 +33,36 @@ FILE-NAME found in ./patches relative to the current file." (define building-on (string-append "--build=" (list-ref (string-split (%current-system) #\-) 0) "-guix-linux-gnu")) +(define (base-binutils target) + (package + (inherit (cross-binutils target)) ;; 2.44 + (version "2.46.0") + (source (origin + (method url-fetch) + (uri (string-append "mirror://gnu/binutils/binutils-" + version ".tar.bz2")) + (sha256 + (base32 + "04nd9vl7c1pxjbc9wh3ckddzhz5g82xyjqq9y9kf171a59im4c8g")))) + (arguments + (substitute-keyword-arguments (package-arguments (cross-binutils target)) + ((#:configure-flags flags) + #~(append #$flags + (list "--enable-gprofng=no"))))) + (native-inputs + (modify-inputs + (package-native-inputs (cross-binutils target)) + (delete "bison"))) + ) +) + (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers base-libc base-gcc) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (base-binutils target)) ;; 1. Build a cross-compiling gcc without targeting any libc, derived ;; from BASE-GCC-FOR-LIBC (xgcc-sans-libc (cross-gcc target @@ -119,7 +137,7 @@ desirable for building Bitcoin Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (binutils-mingw-patches (cross-binutils target))) + (let* ((xbinutils (binutils-mingw-patches (base-binutils target))) (machine (substring target 0 (string-index target #\-))) (pthreads-xlibc (winpthreads-patches (make-mingw-w64 machine #:xgcc (cross-gcc target #:xgcc base-gcc) @@ -147,143 +165,6 @@ chain for " target " development.")) (home-page (package-home-page pthreads-xgcc)) (license (package-license pthreads-xgcc))))) -(define-public python-elfesteem - (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) - (package - (name "python-elfesteem") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/LRGH/elfesteem") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) - (build-system python-build-system) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; PYTHONPATH problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/LRGH/elfesteem") - (synopsis "ELF/PE/Mach-O parsing library") - (description "elfesteem parses ELF, PE and Mach-O files.") - (license license:lgpl2.1)))) - -(define-public python-oscryptotests - (package (inherit python-oscrypto) - (name "python-oscryptotests") - (propagated-inputs - (list python-oscrypto)) - (arguments - `(#:tests? #f - #:phases - (modify-phases %standard-phases - (add-after 'unpack 'hard-code-path-to-libscrypt - (lambda* (#:key inputs #:allow-other-keys) - (chdir "tests") - #t))))))) - -(define-public python-certvalidator - (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) - (package - (name "python-certvalidator") - (version (git-version "0.1" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/certvalidator") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) - (build-system python-build-system) - (propagated-inputs - (list openssl - python-asn1crypto - python-oscrypto - python-oscryptotests)) ;; certvalidator tests import oscryptotests - (arguments - `(#:phases - (modify-phases %standard-phases - (add-after 'unpack 'disable-broken-tests - (lambda _ - (substitute* "tests/test_certificate_validator.py" - (("^(.*)class CertificateValidatorTests" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_crl_client.py" - (("^(.*)def test_fetch_crl" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_ocsp_client.py" - (("^(.*)def test_fetch_ocsp" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_registry.py" - (("^(.*)def test_build_paths" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_hard" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - (substitute* "tests/test_validate.py" - (("^(.*)def test_revocation_mode_soft" line indent) - (string-append indent - "@unittest.skip(\"Disabled by Guix\")\n" - line))) - #t)) - (replace 'check - (lambda _ - (invoke "python" "run.py" "tests") - #t))))) - (home-page "https://github.com/wbond/certvalidator") - (synopsis "Python library for validating X.509 certificates and paths") - (description "certvalidator is a Python library for validating X.509 -certificates or paths. Supports various options, including: validation at a -specific moment in time, whitelisting and revocation checks.") - (license license:expat)))) - -(define-public python-signapple - (let ((commit "85bfcecc33d2773bc09bc318cec0614af2c8e287")) - (package - (name "python-signapple") - (version (git-version "0.2.0" "1" commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/achow101/signapple") - (commit commit))) - (file-name (git-file-name name commit)) - (sha256 - (base32 - "17yqjll8nw83q6dhgqhkl7w502z5vy9sln8m6mlx0f1c10isg8yg")))) - (build-system pyproject-build-system) - (propagated-inputs - (list python-asn1crypto - python-oscrypto - python-certvalidator - python-elfesteem)) - (native-inputs (list python-poetry-core)) - ;; There are no tests, but attempting to run python setup.py test leads to - ;; problems, just disable the test - (arguments '(#:tests? #f)) - (home-page "https://github.com/achow101/signapple") - (synopsis "Mach-O binary signature tool") - (description "signapple is a Python tool for creating, verifying, and -inspecting signatures in Mach-O binaries.") - (license license:expat)))) - (define-public mingw-w64-base-gcc (package (inherit base-gcc) @@ -404,10 +285,9 @@ inspecting signatures in Mach-O binaries.") python-lief) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) - (list zip - (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") + (list (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") nsis-x86_64 - osslsigncode)) + zip)) ((string-contains target "-linux-") (list bison pkg-config @@ -417,6 +297,5 @@ inspecting signatures in Mach-O binaries.") (list clang-toolchain-19 lld-19 (make-lld-wrapper lld-19 #:lld-as-ld? #t) - python-signapple zip)) (else '()))))) diff --git a/contrib/guix/manifest_codesign.scm b/contrib/guix/manifest_codesign.scm new file mode 100644 index 000000000000..652f40ac3f07 --- /dev/null +++ b/contrib/guix/manifest_codesign.scm @@ -0,0 +1,179 @@ +(use-modules ((gnu packages bash) #:select (bash-minimal)) + ((gnu packages compression) #:select (gzip zip)) + ((gnu packages crypto) #:select (osslsigncode)) + ((gnu packages python-build) #:select (python-poetry-core)) + ((gnu packages python-crypto) #:select (python-asn1crypto python-oscrypto)) + ((gnu packages tls) #:select (openssl)) + ((gnu packages version-control) #:select (git-minimal)) + (guix build-system python) + (guix build-system pyproject) + (guix git-download) + ((guix licenses) #:prefix license:) + (guix packages)) + +(define-public python-elfesteem + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) + (package + (name "python-elfesteem") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/LRGH/elfesteem") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) + (build-system python-build-system) + ;; There are no tests, but attempting to run python setup.py test leads to + ;; PYTHONPATH problems, just disable the test + (arguments '(#:tests? #f)) + (home-page "https://github.com/LRGH/elfesteem") + (synopsis "ELF/PE/Mach-O parsing library") + (description "elfesteem parses ELF, PE and Mach-O files.") + (license license:lgpl2.1)))) + +(define-public python-oscryptotests + (package (inherit python-oscrypto) + (name "python-oscryptotests") + (propagated-inputs + (list python-oscrypto)) + (arguments + `(#:tests? #f + #:phases + (modify-phases %standard-phases + (add-after 'unpack 'hard-code-path-to-libscrypt + (lambda* (#:key inputs #:allow-other-keys) + (chdir "tests") + #t))))))) + +(define-public python-certvalidator + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) + (package + (name "python-certvalidator") + (version (git-version "0.1" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/certvalidator") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) + (build-system python-build-system) + (propagated-inputs + (list openssl + python-asn1crypto + python-oscrypto + python-oscryptotests)) ;; certvalidator tests import oscryptotests + (arguments + `(#:phases + (modify-phases %standard-phases + (add-after 'unpack 'disable-broken-tests + (lambda _ + (substitute* "tests/test_certificate_validator.py" + (("^(.*)class CertificateValidatorTests" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_crl_client.py" + (("^(.*)def test_fetch_crl" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_ocsp_client.py" + (("^(.*)def test_fetch_ocsp" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_registry.py" + (("^(.*)def test_build_paths" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_hard" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + (substitute* "tests/test_validate.py" + (("^(.*)def test_revocation_mode_soft" line indent) + (string-append indent + "@unittest.skip(\"Disabled by Guix\")\n" + line))) + #t)) + (replace 'check + (lambda _ + (invoke "python" "run.py" "tests") + #t))))) + (home-page "https://github.com/wbond/certvalidator") + (synopsis "Python library for validating X.509 certificates and paths") + (description "certvalidator is a Python library for validating X.509 +certificates or paths. Supports various options, including: validation at a +specific moment in time, whitelisting and revocation checks.") + (license license:expat)))) + +(define-public python-signapple + (let ((commit "3fab3bb57f227f0dd31007b417683035f5204838")) + (package + (name "python-signapple") + (version (git-version "0.2.0" "1" commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/achow101/signapple") + (commit commit))) + (file-name (git-file-name name commit)) + (sha256 + (base32 + "0qpr78bs50rw79dbihr9ifjq19y6819ih5pn9jd2rbjyifimzf7p")))) + (build-system pyproject-build-system) + (propagated-inputs + (list python-asn1crypto + python-oscrypto + python-certvalidator + python-elfesteem)) + (native-inputs (list python-poetry-core)) + (arguments + ;; There are no tests, but attempting to run python setup.py test leads to + ;; problems, just disable the test + (list #:tests? #f + #:phases + #~(modify-phases %standard-phases + ;; Add a phase to inject OpenSSL paths for oscrypto. + (add-after 'wrap 'wrap-openssl-paths + (lambda* (#:key inputs #:allow-other-keys) + (let ((openssl (assoc-ref inputs "openssl"))) + (wrap-program (string-append #$output "/bin/signapple") + `("SIGNAPPLE_OSCRYPTO_SSL_PATHS" = + (,(string-append openssl "/lib/libcrypto.so" "," openssl "/lib/libssl.so")))))))))) + (home-page "https://github.com/achow101/signapple") + (synopsis "Mach-O binary signature tool") + (description "signapple is a Python tool for creating, verifying, and +inspecting signatures in Mach-O binaries.") + (license license:expat)))) + +(packages->manifest + (append + (list ;; The Basics + bash-minimal + coreutils-minimal + ;; File(system) inspection + findutils + ;; Compression and archiving + tar + gzip + zip + ;; Git + git-minimal) + (let ((target (getenv "HOST"))) + (cond ((string-suffix? "-mingw32" target) + (list osslsigncode)) + ((string-contains target "darwin") + (list python-signapple)) + (else '()))))) diff --git a/contrib/guix/symbol-check.py b/contrib/guix/symbol-check.py index d31d5aa83bcf..86b7965277c9 100755 --- a/contrib/guix/symbol-check.py +++ b/contrib/guix/symbol-check.py @@ -202,7 +202,7 @@ def check_exported_symbols(binary) -> bool: if not symbol.exported: continue name = symbol.name - if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: + if name in IGNORE_EXPORTS: continue print(f'{filename}: export of symbol {name} not allowed!') ok = False diff --git a/contrib/signet/miner b/contrib/signet/miner index f46d88b52e19..5827e4b37a5d 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -66,8 +66,8 @@ def signet_txs(block, challenge): def decode_challenge_psbt(b64psbt): psbt = PSBT.from_base64(b64psbt) - assert len(psbt.tx.vin) == 1 - assert len(psbt.tx.vout) == 1 + assert len(psbt.i) == 1 + assert len(psbt.o) == 1 assert PSBT_SIGNET_BLOCK in psbt.g.map return psbt @@ -333,7 +333,7 @@ class Generate: def gbt(self, bcli, bestblockhash, now): tmpl = json.loads(bcli("getblocktemplate", '{"rules":["signet","segwit"]}')) if tmpl["previousblockhash"] != bestblockhash: - logging.warning("GBT based off unexpected block (%s not %s), retrying", tmpl["previousblockhash"], bci["bestblockhash"]) + logging.warning("GBT based off unexpected block (%s not %s), retrying", tmpl["previousblockhash"], bestblockhash) time.sleep(1) return None diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh deleted file mode 100755 index a532bf74a3d9..000000000000 --- a/contrib/verify-commits/pre-push-hook.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2014-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then - exit 0 -fi - -while read LINE; do - set -- A "$LINE" - if [ "$4" != "refs/heads/master" ]; then - continue - fi - if ! ./contrib/verify-commits/verify-commits.py "$3" > /dev/null 2>&1; then - echo "ERROR: A commit is not signed, can't push" - ./contrib/verify-commits/verify-commits.py - exit 1 - fi -done < /dev/stdin diff --git a/depends/gen_id b/depends/gen_id index 5504deeb0eb1..b9104c43359c 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -28,6 +28,10 @@ # Redirect stderr to stdout exec 2>&1 + # Unset SOURCE_DATE_EPOCH to prevent it from leaking into tool + # outputs and to maximize reuse of the built package cache. + unset SOURCE_DATE_EPOCH + echo "BEGIN ALL" # Include any ID salts supplied via command line diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 214bafafca7c..373ab74e113c 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -6,22 +6,15 @@ LLD_VERSION=711 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers -# We can't just use $(shell command -v clang) because GNU Make handles builtins -# in a special way and doesn't know that `command` is a POSIX-standard builtin -# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. -# At the time of writing, GNU Make v4.2.1 is still being used in supported -# distro releases. -# -# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html -clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") -clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") +clang_prog=$(shell command -v clang) +clangxx_prog=$(shell command -v clang++) -darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar") -darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm") -darwin_OBJCOPY=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objcopy") -darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump") -darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib") -darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip") +darwin_AR=$(shell command -v llvm-ar) +darwin_NM=$(shell command -v llvm-nm) +darwin_OBJCOPY=$(shell command -v llvm-objcopy) +darwin_OBJDUMP=$(shell command -v llvm-objdump) +darwin_RANLIB=$(shell command -v llvm-ranlib) +darwin_STRIP=$(shell command -v llvm-strip) # Flag explanations: # diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk index bcbf2f77b655..240180f2fd2a 100644 --- a/depends/hosts/freebsd.mk +++ b/depends/hosts/freebsd.mk @@ -1,22 +1,15 @@ FREEBSD_VERSION ?= 15.0 FREEBSD_SDK=$(SDK_PATH)/freebsd-$(host)-$(FREEBSD_VERSION)/ -# We can't just use $(shell command -v clang) because GNU Make handles builtins -# in a special way and doesn't know that `command` is a POSIX-standard builtin -# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. -# At the time of writing, GNU Make v4.2.1 is still being used in supported -# distro releases. -# -# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html -clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") -clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") - -freebsd_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar") -freebsd_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm") -freebsd_OBJCOPY=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objcopy") -freebsd_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump") -freebsd_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib") -freebsd_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip") +clang_prog=$(shell command -v clang) +clangxx_prog=$(shell command -v clang++) + +freebsd_AR=$(shell command -v llvm-ar) +freebsd_NM=$(shell command -v llvm-nm) +freebsd_OBJCOPY=$(shell command -v llvm-objcopy) +freebsd_OBJDUMP=$(shell command -v llvm-objdump) +freebsd_RANLIB=$(shell command -v llvm-ranlib) +freebsd_STRIP=$(shell command -v llvm-strip) freebsd_CC=$(clang_prog) --target=$(host) \ diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 7db6afaef6a1..d433636fa4fb 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,7 +1,7 @@ -ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-gcc-posix"),) +ifneq ($(shell command -v $(host)-gcc-posix),) mingw32_CC := $(host)-gcc-posix endif -ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) +ifneq ($(shell command -v $(host)-g++-posix),) mingw32_CXX := $(host)-g++-posix endif diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 59e17f3685a1..5be0e60b623a 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash = 913ca43d49e93d1b158c9862009add1518a4c665e7853b349a6492d $(package)_build_subdir = build define $(package)_set_vars - $(package)_config_opts = -DBOOST_INCLUDE_LIBRARIES="multi_index;signals2;test" + $(package)_config_opts = -DBOOST_INCLUDE_LIBRARIES="multi_index;test" $(package)_config_opts += -DBOOST_TEST_HEADERS_ONLY=ON $(package)_config_opts += -DBOOST_ENABLE_MPI=OFF $(package)_config_opts += -DBOOST_ENABLE_PYTHON=OFF diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index e7f085a377ab..a9ea089d4bf7 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -124,6 +124,22 @@ RPC interface will be abused. security-sensitive operations on a computer whose other programs you trust. +- **RPC Credentials Security Boundary:** Any client with valid RPC credentials + should be treated as having significant control over both the Bitcoin Core node + and the filesystem resources accessible by the `bitcoind` process. RPC commands + can load wallet files from paths that the `bitcoind` process has permission to + access, specify file paths for operations, and potentially gain broader access + than intended. This means that someone with RPC access can potentially compromise + not only the Bitcoin Core node, but also the machine it is running on. Bitcoin Core + provides the `-rpcwhitelist` option to restrict which RPC commands specific users + can access, and `-rpcwhitelistdefault` to control the default behavior for users + without explicit whitelists. However, when using multiple wallets or sharing access + with different users, these should not be considered robust security boundaries, as + users with access to certain commands may still be able to exploit functionality in + unexpected ways. For security-sensitive operations, implement proper system-level + isolation (containers, virtualization, separate user accounts with restricted + permissions) rather than relying solely on RPC access controls. + - **Securing remote network access:** You may optionally allow other computers to remotely control Bitcoin Core by setting the `rpcallowip` and `rpcbind` configuration parameters. These settings are only meant diff --git a/doc/bips.md b/doc/bips.md index 07e5024864c6..ebf6b8fcd7d7 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -49,6 +49,7 @@ BIPs that are implemented by Bitcoin Core: * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). Bech32 addresses are generated by default as of **v0.20.0** ([PR 16884](https://github.com/bitcoin/bitcoin/pull/16884)). * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)). * [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)). +* [`BIP 323`](https://github.com/bitcoin/bips/blob/master/bip-0323.mediawiki): BIP 9 bits 5 to 28 (inclusive) are ignored for soft-fork signalling and unknown soft fork warnings as of **v32.0**. * [`BIP 324`](https://github.com/bitcoin/bips/blob/master/bip-0324.mediawiki): The v2 transport protocol specified by BIP324 and the associated `NODE_P2P_V2` service bit are supported as of **v26.0**, but off by default ([PR 28331](https://github.com/bitcoin/bitcoin/pull/28331)). On by default as of **v27.0** ([PR 29347](https://github.com/bitcoin/bitcoin/pull/29347)). * [`BIP 325`](https://github.com/bitcoin/bips/blob/master/bip-0325.mediawiki): Signet test network is supported as of **v0.21.0** ([PR 18267](https://github.com/bitcoin/bitcoin/pull/18267)). * [`BIP 327`](https://github.com/bitcoin/bips/blob/master/bip-0327.mediawiki): Key aggregation via `musig()` descriptors is supported as of **v30.0** ([PR 31244](https://github.com/bitcoin/bitcoin/pull/31244)). Signing is supported as of **v31.0** ([PR 29675](https://github.com/bitcoin/bitcoin/pull/29675)) diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index daa7fcba0003..e768961de3ee 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -65,7 +65,7 @@ Almost all options can be negated by being specified with a `no` prefix. For exa In general, negating an option is like setting it to `0` if it is a boolean or integer option, and setting it to an empty string or path or list if it is a string or path or list option. -However, there are exceptions to this general rule. For example, it is an error to negate some options (e.g. `-nodatadir` is disallowed), and some negated strings are treated like `"0"` instead of `""` (e.g. `-noproxy` is treated like `-proxy=0`), and some negating some lists can have side effects in addition to clearing the lists (e.g. `-noconnect` disables automatic connections in addition to dropping any manual connections specified previously with `-connect=`). When there are exceptions to the rule, they should either be obvious from context, or should be mentioned in usage documentation. Nonobvious, undocumented exceptions should be reported as bugs. +However, there are exceptions to this general rule. For example, it is an error to negate some options (e.g. `-nodatadir` is disallowed), and some negated strings are treated like `"0"` instead of `""` (e.g. `-noproxy` is treated like `-proxy=0`), and some negating lists can have side effects in addition to clearing the lists (e.g. `-noconnect` disables automatic connections in addition to dropping any manual connections specified previously with `-connect=`). When there are exceptions to the rule, they should either be obvious from context, or should be mentioned in usage documentation. Nonobvious, undocumented exceptions should be reported as bugs. ## Configuration File Path diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index de373d1ecb15..40740d7fb7b3 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -64,7 +64,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkg install libzmq4 ``` diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index f19eb4c5fec3..f50baab4b20b 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -15,7 +15,7 @@ The example commands below use `pkgin`. pkgin install git cmake pkg-config boost libevent ``` -NetBSD currently ships with an older version of `gcc` than is needed to build. You should upgrade your `gcc` and then pass this new version to the configure script. +NetBSD currently ships with an older version of `gcc` than is needed to build. You should upgrade your `gcc` and then pass this new version to the CMake configuration. For example, grab `gcc12`: ``` @@ -82,7 +82,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkgin install zeromq ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 4ea58795a056..5edc56f0bfc1 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -64,7 +64,7 @@ Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` #### Notifications ###### ZeroMQ -Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +Bitcoin Core can provide notifications via ZeroMQ. To compile ZMQ support, install the following dependency and pass `-DWITH_ZMQ=ON` when configuring. ```bash pkg_add zeromq ``` diff --git a/doc/build-osx.md b/doc/build-osx.md index 3c2b0f3a2d97..a6a2d3befe85 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -16,7 +16,9 @@ macOS comes with a built-in Terminal located in: ### 1. Xcode Command Line Tools The Xcode Command Line Tools are a collection of build tools for macOS. -These tools must be installed in order to build Bitcoin Core from source. +Version 16.2 (or higher) of these tools must be +[installed](https://developer.apple.com/documentation/xcode/installing-the-command-line-tools) +in order to build Bitcoin Core from source. To install, run the following command from your terminal: @@ -139,7 +141,7 @@ It is required that you have `python` and `zip` installed. There are many ways to configure Bitcoin Core, here are a few common examples: -##### Wallet (only SQlite) and GUI Support: +##### Wallet and GUI: This enables the GUI. If `sqlite` or `qt` are not installed, this will throw an error. @@ -175,7 +177,7 @@ ctest --test-dir build # Append "-j N" for N parallel tests. ### 3. Deploy (optional) -You can also create a `.zip` containing the `.app` bundle by running the following command: +You can also create a `.zip` containing the `.app` bundle by running the following command: ``` bash cmake --build build --target deploy diff --git a/doc/build-unix.md b/doc/build-unix.md index e499aa9b0f13..f1120d39e3fe 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -1,11 +1,10 @@ -UNIX BUILD NOTES -==================== +# UNIX BUILD NOTES + Some notes on how to build Bitcoin Core in Unix. (For BSD specific instructions, see `build-*bsd.md` in this directory.) -To Build ---------------------- +## To Build ```bash cmake -B build @@ -18,8 +17,7 @@ cmake --install build # Optional ``` See below for instructions on how to [install the dependencies on popular Linux -distributions](#linux-distribution-specific-instructions), or the -[dependencies](#dependencies) section for a complete overview. +distributions](#dependencies). ## Memory Requirements @@ -40,162 +38,46 @@ Finally, clang (often less resource hungry) can be used instead of gcc, which is cmake -B build -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -## Linux Distribution Specific Instructions - -### Ubuntu & Debian - -#### Dependency Build Instructions +## Dependencies -Build requirements for the latest Debian "stable" release, or the latest Ubuntu LTS release: +You can either build from self-compiled [depends](/depends/README.md) or +install the dependencies from your distribution package manager. Dependencies +for additional features in later columns are optional. - sudo apt-get install build-essential cmake pkgconf python3 +| Package manager | Required build dependencies | SQLite (wallet) | Cap'n Proto (IPC) | ZMQ | USDT | Qt and libqrencode (GUI) | +| ----------------------- | --------------------------- | --------------- | ----------------- | --- | ---- | ------------------------ | +| Debian / Ubuntu (`apt`) | `build-essential cmake pkgconf python3 libevent-dev libboost-dev` | `libsqlite3-dev` | `libcapnp-dev capnproto` | `libzmq3-dev` | `systemtap-sdt-dev` | `qt6-base-dev qt6-tools-dev qt6-l10n-tools qt6-tools-dev-tools libgl-dev qt6-wayland libqrencode-dev` | +| Fedora (`dnf`) | `gcc-c++ cmake make python3 libevent-devel boost-devel` | `sqlite-devel` | `capnproto capnproto-devel` | `zeromq-devel` | `systemtap-sdt-devel` | `qt6-qtbase-devel qt6-qttools-devel qt6-qtwayland qrencode-devel` | +| Alpine (`apk`) | `build-base cmake linux-headers pkgconf python3 libevent-dev boost-dev` | `sqlite-dev` | `capnproto capnproto-dev` | `zeromq-dev` | Not supported | `qt6-qtbase-dev qt6-qttools-dev libqrencode-dev` | +| Arch (`pacman`) | `gcc make cmake pkgconf python libevent boost` | `sqlite` | `capnproto` | `zeromq` | `systemtap` | `qt6-base qt6-tools qt6-wayland qrencode` | For Debian "oldstable", or earlier Ubuntu LTS releases, you may need to pick a later compiler version, according to the [dependencies](/doc/dependencies.md) documentation. -Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: - - sudo apt-get install libevent-dev libboost-dev - -SQLite is required for the wallet: - - sudo apt install libsqlite3-dev - To build Bitcoin Core without the wallet, see [*Disable-wallet mode*](#disable-wallet-mode) +and use `-DENABLE_WALLET=OFF` to build without the wallet and skip the SQLite dependency. -Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)): - - sudo apt-get install libcapnp-dev capnproto - +Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)). Compile with `-DENABLE_IPC=OFF` if you do not need IPC functionality. -ZMQ-enabled binaries are compiled with `-DWITH_ZMQ=ON` and require the following dependency: +ZMQ-enabled binaries are compiled with `-DWITH_ZMQ=ON` and require libzmq. - sudo apt-get install libzmq3-dev - -User-Space, Statically Defined Tracing (USDT) dependencies: - - sudo apt install systemtap-sdt-dev +User-Space, Statically Defined Tracing (USDT) requires the systemtap-sdt +development package and must be enabled via `-DWITH_USDT=ON`. GUI dependencies: Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install the necessary parts of Qt, the libqrencode and pass `-DBUILD_GUI=ON`. Skip if you don't intend to use the GUI. - sudo apt-get install qt6-base-dev qt6-tools-dev qt6-l10n-tools qt6-tools-dev-tools libgl-dev - -Additionally, to support Wayland protocol for modern desktop environments: - - sudo apt install qt6-wayland - -The GUI will be able to encode addresses in QR codes unless this feature is explicitly disabled. To install libqrencode, run: - - sudo apt-get install libqrencode-dev +Additionally, install the Qt Wayland platform plugin for modern desktop environments. +The GUI will be able to encode addresses in QR codes and requires libqrencode. Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` option to disable this feature in order to compile the GUI. +### Disable-wallet mode -### Fedora - -#### Dependency Build Instructions - -Build requirements: - - sudo dnf install gcc-c++ cmake make python3 - -Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: - - sudo dnf install libevent-devel boost-devel - -SQLite is required for the wallet: - - sudo dnf install sqlite-devel - -To build Bitcoin Core without the wallet, see [*Disable-wallet mode*](#disable-wallet-mode) - -ZMQ-enabled binaries are compiled with `-DWITH_ZMQ=ON` and require the following dependency: - - sudo dnf install zeromq-devel - -User-Space, Statically Defined Tracing (USDT) dependencies: - - sudo dnf install systemtap-sdt-devel - -Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)): - - sudo dnf install capnproto capnproto-devel - -Compile with `-DENABLE_IPC=OFF` if you do not need IPC functionality. - -GUI dependencies: - -Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install -the necessary parts of Qt, the libqrencode and pass `-DBUILD_GUI=ON`. Skip if you don't intend to use the GUI. - - sudo dnf install qt6-qtbase-devel qt6-qttools-devel - -Additionally, to support Wayland protocol for modern desktop environments: - - sudo dnf install qt6-qtwayland - -The GUI will be able to encode addresses in QR codes unless this feature is explicitly disabled. To install libqrencode, run: - - sudo dnf install qrencode-devel - -Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` option to disable this feature in order to compile the GUI. - -### Alpine - -#### Dependency Build Instructions - -Build requirements: - - apk add build-base cmake linux-headers pkgconf python3 - -Now, you can either build from self-compiled [depends](#dependencies) or install the required dependencies: - - apk add libevent-dev boost-dev - -SQLite is required for the wallet: - - apk add sqlite-dev - -To build Bitcoin Core without the wallet, see [*Disable-wallet mode*](#disable-wallet-mode) - -Cap'n Proto is needed for IPC functionality (see [multiprocess.md](multiprocess.md)): - - apk add capnproto capnproto-dev - -Compile with `-DENABLE_IPC=OFF` if you do not need IPC functionality. - -ZMQ dependencies (provides ZMQ API): - - apk add zeromq-dev - -User-Space, Statically Defined Tracing (USDT) is not supported or tested on Alpine Linux at this time. - -GUI dependencies: - -Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install -the necessary parts of Qt, the libqrencode and pass `-DBUILD_GUI=ON`. Skip if you don't intend to use the GUI. - - apk add qt6-qtbase-dev qt6-qttools-dev - -The GUI will be able to encode addresses in QR codes unless this feature is explicitly disabled. To install libqrencode, run: - - apk add libqrencode-dev - -Otherwise, if you don't need QR encoding support, use the `-DWITH_QRENCODE=OFF` option to disable this feature in order to compile the GUI. - -## Dependencies - -See [dependencies.md](dependencies.md) for a complete overview, and -[depends](/depends/README.md) on how to compile them yourself, if you wish to -not use the packages of your Linux distribution. - -Disable-wallet mode --------------------- When the intention is to only run a P2P node, without a wallet, Bitcoin Core can be compiled in disable-wallet mode with: @@ -204,17 +86,3 @@ be compiled in disable-wallet mode with: In this case there is no dependency on SQLite. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. - -Setup and Build Example: Arch Linux ------------------------------------ -This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: - - pacman --sync --needed capnproto cmake boost gcc git libevent make python sqlite - git clone https://github.com/bitcoin/bitcoin.git - cd bitcoin/ - cmake -B build - cmake --build build - ctest --test-dir build - ./build/bin/bitcoind - ./build/bin/bitcoin help - diff --git a/doc/dependencies.md b/doc/dependencies.md index aac4bb90a080..5e1a19487e90 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,10 +8,12 @@ them using [depends](/depends/README.md). Bitcoin Core requires one of the following compilers. -| Dependency | Minimum required | +| Toolchain | Minimum required | | --- | --- | | [Clang](https://clang.llvm.org) | [17.0](https://github.com/bitcoin/bitcoin/pull/33555) | | [GCC](https://gcc.gnu.org) | [12.1](https://github.com/bitcoin/bitcoin/pull/33842) | +| [Xcode CLT](/doc/build-osx.md) | [16.2](https://github.com/bitcoin/bitcoin/pull/33932) | +| [MSVC](/doc/build-windows-msvc.md) | [18.3](https://github.com/bitcoin/bitcoin/pull/33861) | ## Required diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 8b1e1066082e..e44a4b91093a 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -42,7 +42,7 @@ code. - Constant names are all uppercase, and use `_` to separate words. - Enumerator constants may be `snake_case`, `PascalCase` or `ALL_CAPS`. This is a more tolerant policy than the [C++ Core - Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#renum-caps), which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. See [Internal interface @@ -55,16 +55,13 @@ code. - **Miscellaneous** - `++i` is preferred over `i++`. - - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. - Use a named cast or functional cast, not a C-Style cast. When casting between integer types, use functional casts such as `int(x)` or `int{x}` instead of `(int) x`. When casting between more complex types, use `static_cast`. Use `reinterpret_cast` and `const_cast` as appropriate. - - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-list) where possible. + - Prefer [`list initialization ({})`](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#res-list) where possible. For example `int x{0};` instead of `int x = 0;` or `int x(0);` - - Recursion is checked by clang-tidy and thus must be made explicit. Use - `NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. For function calls a namespace should be specified explicitly, unless such functions have been declared within it. Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be @@ -140,7 +137,18 @@ public: non-optional in-out and output parameters should usually be references, as they cannot be null. -### Coding Style (C++ named arguments) +### Coding Style (clang-tidy rules) + +The clang-tidy tool is used to check some rules. Please refer to the [upstream +documentation](https://clang.llvm.org/extra/clang-tidy/checks/list.html) about +the details and rationale for each rule. + +#### C++ Recursive Functions + +Recursion is checked by clang-tidy and thus must be made explicit. Use +`NOLINTNEXTLINE(misc-no-recursion)` to suppress the check. + +#### C++ named arguments When passing named arguments, use a format that clang-tidy understands. The argument names can otherwise not be verified by clang-tidy. @@ -156,7 +164,7 @@ int main() } ``` -### Running clang-tidy +#### Running clang-tidy To run clang-tidy on Ubuntu/Debian, install the dependencies: @@ -164,7 +172,7 @@ To run clang-tidy on Ubuntu/Debian, install the dependencies: apt install clang-tidy clang ``` -Configure with clang as the compiler: +Configure with clang as the compiler with the below command that should create a `compile_commands.json` file within the build directory: ```sh cmake -B build -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON @@ -172,22 +180,25 @@ cmake -B build -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_EXP The output is denoised of errors from external dependencies. -To run clang-tidy on all source files: +To run clang-tidy on all source files using the checks mentioned in the `./src/.clang-tidy` file: ```sh ( cd ./src/ && run-clang-tidy -p ../build -j $(nproc) ) ``` +To run clang-tidy on one file: +```sh +( cd ./src/ && run-clang-tidy -p ../build -j $(nproc) ./path/to/single_file.cpp ) +``` + +Optionally, append the `run-clang-tidy` command with the `-quiet` option to suppress printing of statistics and ignored warnings that can clutter the output. The `-fix` option also comes in handy to apply the fixes suggested by the tool but need to ensure that unrelated changes in the file are not committed. + To run clang-tidy on the changed source lines: ```sh git diff | ( cd ./src/ && clang-tidy-diff -p2 -path ../build -j $(nproc) ) ``` -## Coding Style (Python) - -Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). - ## Coding Style (Doxygen-compatible comments) Bitcoin Core uses [Doxygen](https://www.doxygen.nl/) to generate its official documentation. @@ -291,6 +302,10 @@ Linux: `sudo apt install doxygen graphviz` MacOS: `brew install doxygen graphviz` +## Coding Style (Python) + +Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). + ## Development tips and tricks ### Compiling for debugging @@ -379,22 +394,6 @@ other input. assumption may or may not result in a slightly degraded user experience, but it is safe to continue program execution. -### Valgrind suppressions file - -Valgrind is a programming tool for memory debugging, memory leak detection, and -profiling. The repo contains a Valgrind suppressions file -([`valgrind.supp`](/test/sanitizer_suppressions/valgrind.supp)) -which includes known Valgrind warnings in our dependencies that cannot be fixed -in-tree. Example use: - -```shell -$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp build/bin/test_bitcoin -$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp --leak-check=full \ - --show-leak-kinds=all build/bin/test_bitcoin --log_level=test_suite -$ valgrind -v --leak-check=full build/bin/bitcoind -printtoconsole -$ ./build/test/functional/test_runner.py --valgrind -``` - ### Compiling for test coverage #### Using LCOV @@ -457,7 +456,8 @@ LLVM_PROFILE_FILE="$(pwd)/build/raw_profile_data/%m_%p.profraw" ctest --test-dir LLVM_PROFILE_FILE="$(pwd)/build/raw_profile_data/%m_%p.profraw" build/test/functional/test_runner.py # Append "-j N" here for N parallel jobs # Merge all the raw profile data into a single file -find build/raw_profile_data -name "*.profraw" | xargs llvm-profdata merge -o build/coverage.profdata +find build/raw_profile_data -name "*.profraw" > build/raw_profile_data_files.txt +llvm-profdata merge -f build/raw_profile_data_files.txt -o build/coverage.profdata ``` > **Note:** The "counter mismatch" warning can be safely ignored, though it can be resolved by updating to Clang 19. @@ -531,22 +531,32 @@ The generated coverage report can be accessed at `build/coverage_report/index.ht The [`include-what-you-use`](https://github.com/include-what-you-use/include-what-you-use) tool (IWYU) helps to enforce the source code organization [policy](#source-code-organization) in this repository. -To ensure consistency, it is recommended to run the IWYU CI job locally rather than running the tool directly. +To reproduce the IWYU CI job locally, run: +```bash +env -i HOME="$HOME" PATH="$PATH" USER="$USER" MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_native_iwyu.sh" ./ci/test_run_all.sh || echo "IWYU failed" +``` In some cases, IWYU might suggest headers that seem unnecessary at first glance, but are actually required. For example, a macro may use a symbol that requires its own include. Another example is passing a string literal to a function that accepts a `std::string` parameter. An implicit conversion occurs at the callsite using the `std::string` constructor, which makes the corresponding header required. We accept these suggestions as is. -Use `IWYU pragma: export` very sparingly, as this enforces transitive inclusion of headers -and undermines the specific purpose of IWYU. +If the provided IWYU CI job still produces a false positive, reduce it to a minimal reproducer and report it upstream. + +Use IWYU pragmas sparingly. + +Use `IWYU pragma: keep` only as a narrow workaround when needed. + +Use `IWYU pragma: associated` only when IWYU cannot infer the intended associated header. + +Use `IWYU pragma: export` very sparingly, as this enforces transitive inclusion of headers and undermines the specific purpose of IWYU. The acceptable cases for using `IWYU pragma: export` are: 1. Facade headers. For example, see [`compat/compat.h`](/src/compat/compat.h). 2. Drop-in replacement headers. For example, see [`util/time.h`](/src/util/time.h). 3. Presenting a complete interface across multiple headers. -A comment explaining the rationale is required for every use of `IWYU pragma: export`. +For IWYU pragmas, prefer adding a nearby source comment that explains why the annotation is needed. ### Performance profiling with perf @@ -591,6 +601,21 @@ or using a graphical tool like [Hotspot](https://github.com/KDAB/hotspot). See the functional test documentation for how to invoke perf within tests. +### Valgrind + +Valgrind is a programming tool for memory debugging, memory leak detection, and +profiling. The repo contains a Valgrind suppressions file +([`valgrind.supp`](/test/sanitizer_suppressions/valgrind.supp)) +which includes known Valgrind warnings in our dependencies that cannot be fixed +in-tree. Example use: + +```shell +$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp build/bin/test_bitcoin +$ valgrind --suppressions=test/sanitizer_suppressions/valgrind.supp --leak-check=full \ + --show-leak-kinds=all build/bin/test_bitcoin --log_level=test_suite +$ valgrind -v --leak-check=full build/bin/bitcoind -printtoconsole +$ ./build/test/functional/test_runner.py --valgrind +``` ### Sanitizers @@ -646,6 +671,11 @@ Additional resources: * [GCC Instrumentation Options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) * [Google Sanitizers Wiki](https://github.com/google/sanitizers/wiki) +# Development guidelines + +A few non-style-related recommendations for developers, as well as points to +pay attention to for reviewers of Bitcoin Core code. + ## Locking/mutex usage notes The code is multi-threaded and uses mutexes and the @@ -664,62 +694,57 @@ and its `cs_KeyStore` lock for example). ## Threads -- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#a0ddf1224851353fc92bfbff6f499fa97) +- [Main thread (`bitcoind`)](https://doxygen.bitcoincore.org/bitcoind_8cpp.html#main) : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and shutting down the application. -- [Init load (`b-initload`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) +- [Init load (`b-initload`)](https://doxygen.bitcoincore.org/init_8cpp.html#initload) : Performs various loading tasks that are part of init but shouldn't block the node from being started: external block import, reindex, reindex-chainstate, main chain activation, spawn indexes background sync threads and mempool load. -- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#a6e7fa51d3a25e7cb65446d4b50e6a987) +- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#checkqueue) : Parallel script validation threads for transactions in blocks. -- [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) +- [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http) : Libevent thread to listen for RPC and REST connections. -- [HTTP worker threads (`b-http_pool_x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#a2ad0a49dc9b5e8117c0dee98c24187d8) +- [HTTP worker threads (`b-http_pool_x`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#http_pool) : Threads to service RPC and REST requests. -- [Indexer threads (`b-txindex`, etc)](https://doxygen.bitcoincore.org/class_base_index.html#a96a7407421fbf877509248bbe64f8d87) +- [Indexer threads (`b-txindex`, etc)](https://doxygen.bitcoincore.org/class_base_index.html#index_sync) : One thread per indexer. -- [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#a14d2800815da93577858ea078aed1fba) +- [SchedulerThread (`b-scheduler`)](https://doxygen.bitcoincore.org/class_c_scheduler.html#scheduler) : Does asynchronous background tasks like dumping wallet contents, dumping addrman and running asynchronous validationinterface callbacks. -- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a52a3efff23634500bb42c6474f306091) +- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/class_tor_controller.html#torcontrol) : Libevent thread for tor connections. - Net threads: - - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#aacdbb7148575a31bb33bc345e2bf22a9) + - [ThreadMessageHandler (`b-msghand`)](https://doxygen.bitcoincore.org/class_c_connman.html#msghand) : Application level message handling (sending and receiving). Almost all net_processing and validation logic runs on this thread. - - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) + - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#dnsseed) : Loads addresses of peers from the DNS. - - ThreadMapPort (`b-mapport`) + - [ThreadMapPort (`b-mapport`)](https://doxygen.bitcoincore.org/mapport_8cpp.html#mapport) : Universal plug-and-play startup/shutdown. - - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) + - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#net) : Sends/Receives data from peers on port 8333. - - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a0b787caf95e52a346a2b31a580d60a62) + - [ThreadOpenAddedConnections (`b-addcon`)](https://doxygen.bitcoincore.org/class_c_connman.html#addcon) : Opens network connections to added nodes. - - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) + - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#opencon) : Initiates new connections to peers. - - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#a57787b4f9ac847d24065fbb0dd6e70f8) + - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#i2paccept) : Listens for and accepts incoming I2P connections through the I2P SAM proxy. -# Development guidelines - -A few non-style-related recommendations for developers, as well as points to -pay attention to for reviewers of Bitcoin Core code. - ## General Bitcoin Core - New features should be exposed on RPC first, then can be made available in the GUI. @@ -736,13 +761,12 @@ logging messages. They should be used as follows: most of the time, and it should be used for log messages that are useful for debugging and can reasonably be enabled on a production system (that has sufficient free storage space). They will be logged - if the program is started with `-debug=category` or `-debug=1`. + if the program is started with `-debug=category` or `-debug=1`, or + the category is enabled through the `logging` RPC. - `LogInfo(fmt, params...)` should only be used rarely, e.g. for startup messages or for infrequent and important events such as a new block tip - being found or a new outbound connection being made. These log messages - are unconditional, so care must be taken that they can't be used by an - attacker to fill up storage. + being found or a new outbound connection being made. - `LogError(fmt, params...)` should be used in place of `LogInfo` for severe problems that require the node (or a subsystem) to shut down @@ -764,6 +788,13 @@ Note that the format strings and parameters of `LogDebug` and `LogTrace` are only evaluated if the logging category is enabled, so you must be careful to avoid side-effects in those expressions. +While `LogInfo`, `LogWarning` and `LogError` messages should be rare, +in case there are circumstances where they are not, those messages +are automatically rate-limited to prevent potential disk-filling +attacks. For the cases where this protection is undesirable, +rate-limiting can be avoided with the `util::log::NO_RATE_LIMIT` tag, eg +`LogInfo(util::log::NO_RATE_LIMIT, "UpdateTip: new best=%s ...",...)`. + ## General C++ For general C++ guidelines, you may refer to the [C++ Core @@ -772,7 +803,7 @@ Guidelines](https://isocpp.github.io/CppCoreGuidelines/). Common misconceptions are clarified in those sections: - Passing (non-)fundamental types in the [C++ Core - Guideline](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-conventional). + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#rf-conventional). - If you use the `.h`, you must link the `.cpp`. @@ -796,7 +827,7 @@ Common misconceptions are clarified in those sections: - Do not compare an iterator from one data structure with an iterator of another data structure (even if of the same type). - - *Rationale*: Behavior is undefined. In C++ parlor this means "may reformat + - *Rationale*: Behavior is undefined. In C++ parlance this means "may reformat the universe", in practice this has resulted in at least one hard-to-debug crash bug. - Watch out for out-of-bounds vector access. `&vch[vch.size()]` is illegal, @@ -1356,6 +1387,46 @@ A few guidelines for modifying existing RPC interfaces: - *Rationale*: Changes in RPC JSON structure can break downstream application compatibility. Implementation of `deprecatedrpc` provides a grace period for downstream applications to migrate. Release notes provide notification to downstream users. +## Feature deprecation and removal process + +Bitcoin Core uses a structured process for deprecating and removing features to give +downstream users and applications time to migrate. + +### General principles + +- The minimum deprecation **grace period** for a feature that is going to be removed is one + major release. +- Any deprecation or removal must come with a release note. + +### RPC methods and fields + +- To deprecate an entire RPC method, gate the old behavior behind `-deprecatedrpc=`. + Deprecated features should remain accessible via this flag during the grace period so + downstream users are not immediately broken. +- The RPC help text must mention the deprecation and the `-deprecatedrpc=` flag + that re-enables it. For example: + ``` + "\nDeprecated in v25.0, use the `newfoo` RPC instead. Start bitcoind with" + " `-deprecatedrpc=foo` to continue using this RPC.\n" + ``` + +### Startup options + +- To deprecate a startup option, emit a warning via `LogWarning` or `InitWarning` when the + option is used, so users are notified at startup. +- Update the option's help text to indicate it is deprecated and, if applicable, will be + removed in a future release. + +### REST interface + +- Deprecated REST endpoints or behaviors should be documented in `doc/REST-interface.md` + with the version they were deprecated. + +### ZMQ + +- Deprecated ZMQ topics or behaviors should be documented in `doc/zmq.md` with the version + they were deprecated. + ## Internal interface guidelines Internal interfaces between parts of the codebase that are meant to be @@ -1417,9 +1488,9 @@ communication: using TipChangedFn = std::function; virtual std::unique_ptr handleTipChanged(TipChangedFn fn) = 0; - // Bad: returns boost connection specific to local process + // Bad: returns btcsignals connection specific to local process using TipChangedFn = std::function; - virtual boost::signals2::scoped_connection connectTipChanged(TipChangedFn fn) = 0; + virtual btcsignals::scoped_connection connectTipChanged(TipChangedFn fn) = 0; ``` - Interface methods should not be overloaded. diff --git a/doc/external-signer.md b/doc/external-signer.md index c1b6b20ce017..ee1b68bb89f6 100644 --- a/doc/external-signer.md +++ b/doc/external-signer.md @@ -2,6 +2,8 @@ Bitcoin Core can be launched with `-signer=` where `` is an external tool which can sign transactions and perform other functions. For example, it can be used to communicate with a hardware wallet. +Interaction with external signer uses [Partially Signed Bitcoin Transaction (PSBT)](psbt.md). + ## Example usage The following example is based on the [HWI](https://github.com/bitcoin-core/HWI) tool. Version 2.0 or newer is required. Although this tool is hosted under the Bitcoin Core GitHub organization and maintained by Bitcoin Core developers, it should be used with caution. It is considered experimental and has far less review than Bitcoin Core itself. Be particularly careful when running tools such as these on a computer with private keys on it. @@ -11,7 +13,7 @@ When using a hardware wallet, consult the manufacturer website for (alternative) Start Bitcoin Core: ```sh -$ bitcoind -signer=../HWI/hwi.py +bitcoind -signer=../HWI/hwi.py ``` `bitcoin node` can also be substituted for `bitcoind`. @@ -24,14 +26,19 @@ Follow the hardware manufacturers instructions for the initial device setup, as Get a list of signing devices / services: +```sh +bitcoin-cli enumeratesigners +``` + ``` -$ bitcoin-cli enumeratesigners { "signers": [ { - "fingerprint": "c8df832a" + "fingerprint": "c8df832a", + "name": "trezor_t" } -] + ] +} ``` The master key fingerprint is used to identify a device. @@ -39,92 +46,124 @@ The master key fingerprint is used to identify a device. Create a wallet, this automatically imports the public keys: ```sh -$ bitcoin-cli createwallet "hww" true true "" true true true +bitcoin rpc createwallet wallet_name="hww2" disable_private_keys=true descriptors=true external_signer=true ``` -`bitcoin rpc` can also be substituted for `bitcoin-cli`. +Creation of the external wallet can be confirmed with `getwalletinfo`, which will report `"external_signer": true`. These commands can also be executed using `bitcoin-qt` Debug Console instead of using `bitcoin rpc` or `bitcoin-cli`. ### Verify an address Display an address on the device: ```sh -$ bitcoin-cli -rpcwallet= getnewaddress -$ bitcoin-cli -rpcwallet= walletdisplayaddress
+bitcoin-cli -rpcwallet= getnewaddress +bitcoin-cli -rpcwallet= walletdisplayaddress
``` Replace `
` with the result of `getnewaddress`. ### Spending -Under the hood this uses a [Partially Signed Bitcoin Transaction](psbt.md). +Under the hood this uses a [PSBT (Partially Signed Bitcoin Transaction)](psbt.md). ```sh -$ bitcoin-cli -rpcwallet= sendtoaddress
+bitcoin-cli -rpcwallet= sendtoaddress
``` -This prompts your hardware wallet to sign, and fail if it's not connected. If successful -it automatically broadcasts the transaction. +This constructs a PSBT and prompts your external signer to sign (will fail if it's not connected). If successful, Bitcoin Core finalizes and broadcasts the transaction. -```sh -{"complete": true, "txid": } +``` +{"complete": true, "txid": ""} ``` ## Signer API -In order to be compatible with Bitcoin Core any signer command should conform to the specification below. This specification is subject to change. Ideally a BIP should propose a standard so that other wallets can also make use of it. +In order to be compatible with Bitcoin Core, any signer command should conform to the specification below. This specification is subject to change. Ideally a BIP should propose a standard so that other wallets can also make use of it. Prerequisite knowledge: * [Output Descriptors](descriptors.md) * Partially Signed Bitcoin Transaction ([PSBT](psbt.md)) +### Flag `--chain ` (required) + +With `` one of `main`, `test`, `signet`, `regtest`, `testnet4`. + +### Flag `--stdin` (required) + +Indicate that (sub)command should be received over stdin and results returned in response to that. `--stdin` is a global flag, it is not used for all subcommands. + +All subcommands SHOULD support both +- being called as commandline arguments; or +- being written to _external-signer_ process directly through stdin (with `--stdin`). + +Usage: +```sh + --stdin … +``` + +``` +[…command and arguments written to stdin…] +``` + +Note: remember that shell-expansion is not available on _stdin_. Consequently, commands such as `signtx`, may write their arguments in either quoted or unquoted form. + +### Flag `--fingerprint ` (required) + +With `` being the hexadecimal 8-symbol identifier for a wallet. + +Commands will specify a fingerprint as an identifier for external-signer wallet. + ### `enumerate` (required) Usage: + +```sh + enumerate +``` + ``` -$ enumerate [ { - "fingerprint": "00000000" + "fingerprint": "00000000", + "name": "trezor_t" } ] ``` -The command MUST return an (empty) array with at least a `fingerprint` field. +The command MUST return an array, possibly empty, of entries that contain at least a `fingerprint` field. A future extension could add an optional return field with device capabilities. Perhaps a descriptor with wildcards. For example: `["pkh("44'/0'/$'/{0,1}/*"), sh(wpkh("49'/0'/$'/{0,1}/*")), wpkh("84'/0'/$'/{0,1}/*")]`. This would indicate the device supports legacy, wrapped SegWit and native SegWit. In addition it restricts the derivation paths that can used for those, to maintain compatibility with other wallet software. It also indicates the device, or the driver, doesn't support multisig. A future extension could add an optional return field `reachable`, in case `` knows a signer exists but can't currently reach it. -### `signtransaction` (required) +### `signtx` (required) -Usage: -``` -$ --fingerprint= (--testnet) signtransaction -base64_encode_signed_psbt -``` +`signtx` indicates a PSBT signing-request, followed by Base64-encoded PSBT. Quotes are optional. -The command returns a psbt with any signatures. +This command reads request `` and MUST return a JSON object. On success, it MUST contain `{"psbt": ""}` updated to include any new signatures. On failure, it SHOULD contain `{"error": ""}`. Presence of a key `error` with value `null` is not considered an error. -The `psbt` SHOULD include bip32 derivations. The command SHOULD fail if none of the bip32 derivations match a key owned by the device. +PSBT SHOULD include BIP32 derivations. The command SHOULD fail if none of the BIP32 derivations match a key owned by the device. The command SHOULD fail if the user cancels. -The command MAY complain if `--testnet` is set, but any of the BIP32 derivation paths contain a coin type other than `1h` (and vice versa). +The command MAY complain if `--chain` is set to a test-network, but any of the BIP32 derivation paths contain a coin type other than `1h` (and vice versa). ### `getdescriptors` (optional) Usage: -``` -$ --fingerprint= (--testnet) getdescriptors +```sh + --fingerprint= getdescriptors ``` Returns descriptors supported by the device. Example: +```sh + --fingerprint=00000000 getdescriptors +``` + ``` -$ --fingerprint=00000000 --testnet getdescriptors { "receive": [ "pkh([00000000/44h/0h/0h]xpub6C.../0/*)#fn95jwmg", @@ -142,14 +181,14 @@ $ --fingerprint=00000000 --testnet getdescriptors ### `displayaddress` (optional) Usage: -``` - --fingerprint= (--testnet) displayaddress --desc descriptor +```sh + --fingerprint= displayaddress --desc ``` Example, display the first native SegWit receive address on Testnet: -``` - --fingerprint=00000000 --testnet displayaddress --desc "wpkh([00000000/84h/1h/0h]tpubDDUZ..../0/0)" +```sh + --chain test --fingerprint=00000000 displayaddress --desc "wpkh([00000000/84h/1h/0h]tpubDDUZ..../0/0)" ``` The command MUST be able to figure out the address type from the descriptor. @@ -157,11 +196,11 @@ The command MUST be able to figure out the address type from the descriptor. The command MUST return an object containing `{"address": "[the address]"}`. As a sanity check, for devices that support this, it SHOULD ask the device to derive the address. -If contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device. +If `` contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device. -If contains an xpub, the command MUST fail if it does not match the xpub known by the device. +If `` contains an xpub, the command MUST fail if it does not match the xpub known by the device. -The command MAY complain if `--testnet` is set, but the BIP32 coin type is not `1h` (and vice versa). +The command MAY complain if `--chain` is set to a test-network, but the BIP32 coin-type is not `1h` (and vice versa). ## How Bitcoin Core uses the Signer API @@ -169,10 +208,12 @@ The `enumeratesigners` RPC simply calls ` enumerate`. The `createwallet` RPC calls: -* ` --fingerprint=00000000 getdescriptors 0` +* ` --chain --fingerprint=00000000 getdescriptors --account 0` -It then imports descriptors for all support address types, in a BIP44/49/84 compatible manner. +It then imports descriptors for all supported address types, in a BIP44/49/84/86 compatible manner. The `walletdisplayaddress` RPC reuses some code from `getaddressinfo` on the provided address and obtains the inferred descriptor. It then calls ` --fingerprint=00000000 displayaddress --desc=`. +For external-signer wallets, spending uses `send` or `sendall`. Bitcoin Core builds a PSBT, calls the signer via stdin with `signtx`, and if signatures are sufficient, finalizes and broadcasts the transaction. If the signer is not connected or cancels, the call fails with an error. For fee-bumping on such wallets, use `psbtbumpfee` to involve an external signer. + `sendtoaddress` and `sendmany` check `inputs->bip32_derivs` to see if any inputs have the same `master_fingerprint` as the signer. If so, it calls ` --fingerprint=00000000 signtransaction `. It waits for the device to return a (partially) signed psbt, tries to finalize it and broadcasts the transaction. diff --git a/doc/files.md b/doc/files.md index 8a11e08913cd..e8f286929264 100644 --- a/doc/files.md +++ b/doc/files.md @@ -165,5 +165,5 @@ This table describes the files installed by Bitcoin Core across different platfo ## Filesystem recommendations -When choosing a filesystem for the data directory (`datadir`) or blocks directory (`blocksdir`) on **macOS**,the `exFAT` filesystem should be avoided. +When choosing a filesystem for the data directory (`datadir`) or blocks directory (`blocksdir`) on **macOS**, the `exFAT` filesystem should be avoided. There have been multiple reports of database corruption and data loss when using this filesystem with Bitcoin Core, see [Issue #31454](https://github.com/bitcoin/bitcoin/issues/31454) for more details. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 4b108862cd9c..253f3f12ea23 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -14,6 +14,8 @@ $ FUZZ=process_message build_fuzz/bin/fuzz ``` One can use `--preset=libfuzzer-nosan` to do the same without common sanitizers enabled. +Note that this preset uses a different build directory, so replace `build_fuzz` with +`build_fuzz_nosan` in the build and run commands above. See [further](#run-without-sanitizers-for-increased-throughput) for more information. There is also a runner script to execute all fuzz targets. Refer to diff --git a/doc/i2p.md b/doc/i2p.md index 624b651f62a9..2877c1e570ec 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -86,7 +86,7 @@ address is used for making outbound connections and accepting inbound connections. In the I2P network, the receiver of an inbound connection sees the address of -the initiator. This is unlike the Tor network, where the recipient does not +the initiator. This is unlike the Tor network, where the recipient does not know who is connecting to it. If your node is configured by setting `-i2pacceptincoming=0` to not accept diff --git a/doc/release-notes-21283.md b/doc/release-notes-21283.md new file mode 100644 index 000000000000..fb5494a1f73b --- /dev/null +++ b/doc/release-notes-21283.md @@ -0,0 +1,6 @@ +Updated RPCs +------------ + +- `createpsbt`, `walletcreatepsbt`, `converttopsbt`, and `psbtbumpfee` + will now default to creating version 2 PSBTs. An optional `psbt_version` + argument is added to these RPCs which allows specifying the version of PSBT to create. diff --git a/doc/release-notes-26201.md b/doc/release-notes-26201.md index 66c9da540611..108b1bfd47eb 100644 --- a/doc/release-notes-26201.md +++ b/doc/release-notes-26201.md @@ -1,4 +1,9 @@ RPC --- -- 'taproot' has been removed from 'getdeploymentinfo' because its historical activation height is no longer used anywhere in the codebase. (#26201) +- 'taproot' has been removed from 'getdeploymentinfo' because its historical + activation height is no longer used anywhere in the codebase. + Applications that rely on `deployments.taproot` to detect Taproot support + should either assume Taproot to be always active (since Bitcoin Core v24.0), + or check for `TAPROOT` in the `script_flags` array returned by + `getdeploymentinfo` (since Bitcoin Core v31.0). (#26201) diff --git a/doc/release-notes-29136.md b/doc/release-notes-29136.md new file mode 100644 index 000000000000..af074a0b30ad --- /dev/null +++ b/doc/release-notes-29136.md @@ -0,0 +1,6 @@ +Wallet +------ +- A new RPC `addhdkey` is added which allows a BIP 32 extended key to be added to the wallet without + needing to import it as part of a separate descriptor. This key will not be used to produce any + output scripts unless it is explicitly imported as part of a separate descriptor independent of + the `addhdkey` RPC. diff --git a/doc/release-notes-32220.md b/doc/release-notes-32220.md new file mode 100644 index 000000000000..c25271571554 --- /dev/null +++ b/doc/release-notes-32220.md @@ -0,0 +1,9 @@ +### Build System + +- The undocumented `BITCOIN_GENBUILD_NO_GIT` environment variable is no longer + required and has been removed. The build system now automatically detects + when it is being built from a source archive or as a subproject of a git-aware + parent project and skips git metadata fetching. + + Users who need to disable git execution explicitly can still do so by + configuring with `-DCMAKE_DISABLE_FIND_PACKAGE_Git=ON`. diff --git a/doc/release-notes-33920.md b/doc/release-notes-33920.md new file mode 100644 index 000000000000..a47dff088931 --- /dev/null +++ b/doc/release-notes-33920.md @@ -0,0 +1,4 @@ +New RPCs +-------- + +A new `exportasmap` RPC writes the ASMap data embedded at build time to a file. diff --git a/doc/release-notes-33966.md b/doc/release-notes-33966.md new file mode 100644 index 000000000000..7251a13d52d1 --- /dev/null +++ b/doc/release-notes-33966.md @@ -0,0 +1,9 @@ +Mining +------ + +- The IPC mining interface now rejects out-of-range block template options + instead of silently clamping them, such as oversized reserved block weight or + coinbase sigops limits. (#33966) + +- The `-blockmaxweight` startup option is now rejected when it is lower than + `-blockreservedweight`, instead of being silently clamped. (#33966) diff --git a/doc/release-notes-34544.md b/doc/release-notes-34544.md new file mode 100644 index 000000000000..bcc3e38f3c46 --- /dev/null +++ b/doc/release-notes-34544.md @@ -0,0 +1,7 @@ +Wallet +------ + +- Wallets names that are relative paths including `..` and `.` elements, and + wallets named `/` are no longer allowed. Any users that depended on this + behavior can instead use absolute paths to their wallet or move their wallet + to a safer path. diff --git a/doc/release-notes-34779.md b/doc/release-notes-34779.md new file mode 100644 index 000000000000..7f1f5099d064 --- /dev/null +++ b/doc/release-notes-34779.md @@ -0,0 +1,5 @@ +Logging +------- + +- BIP 9 bits 5 to 28 inclusive are now ignored for soft fork signaling, as per BIP 323. We won't + warn about unknown deployments when receiving blocks that set any of those bits in their version. diff --git a/doc/release-notes-34911.md b/doc/release-notes-34911.md new file mode 100644 index 000000000000..43520296726d --- /dev/null +++ b/doc/release-notes-34911.md @@ -0,0 +1,12 @@ +### Mempool + +mempoolfullrbf=1 behaviour has been the default since v28 and the argument has +been removed since v29 subsequently. The `getmempoolinfo` RPC stops returning the +deprecated `fullrbf` key in the response unless the user requests it via the +`-deprecatedrpc=fullrbf` node argument. + +Also, the `bip125-replaceable` key is removed from the mempool RPCs +responses (because it, too, has been deprecated since v29) unless the user +requests it via `-deprecatedrpc=bip125` node argument. Affected mempool RPCs +are `getrawmempool`, `getmempoolancestors`, `getmempooldescendants`, and +`getmempoolentry`. diff --git a/doc/release-notes-34917.md b/doc/release-notes-34917.md new file mode 100644 index 000000000000..cd79418c0b6e --- /dev/null +++ b/doc/release-notes-34917.md @@ -0,0 +1,9 @@ +RPC and Startup Option +------------ +The `bip125-replaceable` key in the wallet transaction RPCs such +as `listtransactions`, `listsinceblock`, and `gettransaction` is +marked as deprecated. Users still have the option to retrieve this +key by passing the `-deprecatedrpc=bip125` startup option. Also, +the `-walletrbf` startup option has been marked as deprecated and +will be fully removed in the next release. Using this option emits +a warning in the logs. diff --git a/doc/release-notes-35267.md b/doc/release-notes-35267.md new file mode 100644 index 000000000000..ff9ad4550ef9 --- /dev/null +++ b/doc/release-notes-35267.md @@ -0,0 +1,4 @@ +RPC +--- + +The `getprivatebroadcastinfo` and `abortprivatebroadcast` RPCs now return an error `-32601`, "Method not found", when `-privatebroadcast` is not enabled at startup. diff --git a/doc/release-notes-35319.md b/doc/release-notes-35319.md new file mode 100644 index 000000000000..6b7299693105 --- /dev/null +++ b/doc/release-notes-35319.md @@ -0,0 +1,20 @@ +P2P and network changes +----------------------- + +- Fix a possible leak of the originator's IP address for transactions sent with +`sendrawtransaction` RPC when `-privatebroadcast=1`. When Bitcoin Core connects +to a peer, if that peer has been advertised to support P2P protocol v2, then +Bitcoin Core tries to use the v2 protocol and if that fails it retries the +connection using the v1 protocol. When the private broadcast is about to send a +transaction to an IPv4 or IPv6 peer it overrides the normal proxy selection and +forces the connection through the Tor proxy (and thus through the Tor network, +protecting the sender's IP address). However if v2 protocol is tried and it +fails, then the v1 retry connection would be made disregarding the "override +proxy" request, possibly making an IPv4 or IPv6 direct connection. In other +words, for this to happen the following must be true: + 1. An IPv4 or IPv6 address has been advertised to the sender, indicating v2 + support. + 2. The sender must have Tor configured. + 3. The sender's configuration must be such that connections to IPv4 or IPv6 + are made directly (no `-proxy=` is used for IPv4 or IPv6). + 4. The recipient does not support v2 (the flags from 1. are bogus). (#35319) diff --git a/doc/release-notes-empty-template.md b/doc/release-notes-empty-template.md index 86dafb36ef96..70990c73863f 100644 --- a/doc/release-notes-empty-template.md +++ b/doc/release-notes-empty-template.md @@ -20,6 +20,15 @@ To receive security and update notifications, please subscribe to: +With the release of this new major version, versions *version minus 3* and +older are at "End of Life" and will no longer receive updates. + +In accordance with the security policy, we will in two weeks disclose: + +* Medium and high severity vulnerabilities fixed in *version minus 2*. There are N of these. + +* Low severity vulnerabilities fixed in *version*. There are M of these. + How to Upgrade ============== diff --git a/doc/release-notes/release-notes-31.0.md b/doc/release-notes/release-notes-31.0.md new file mode 100644 index 000000000000..7246702a9055 --- /dev/null +++ b/doc/release-notes/release-notes-31.0.md @@ -0,0 +1,382 @@ +v31.0 Release Notes +=================== + +Bitcoin Core version 31.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + + +With the release of this new major version, versions `28.x` and +older are at "End of Life" and will no longer receive updates. + +In accordance with the security policy, we will in two weeks disclose: + +* Medium and high severity vulnerabilities fixed in `29.0`. There is one of these. + +* Low severity vulnerabilities fixed in `31.0`. There are none of these. + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the installer +(on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) or +`bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be +migrated. Old wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and tested on the following operating systems or +newer: Linux Kernel 3.17, macOS 14, and Windows 10 (version 1903). Bitcoin Core +should also work on most other Unix-like systems but is not as frequently tested +on them. It is not recommended to use Bitcoin Core on unsupported systems. + +Notable changes +=============== + +The default `-dbcache` value has been increased to 1024 MiB from 450 MiB on +systems where at least 4096 MiB of RAM is detected. This improves performance +but increases memory usage. On some systems (for example when running in +containers), the detected RAM may exceed the memory actually available, which +can lead to out-of-memory conditions. To maintain the previous behavior, set +`-dbcache=450`. See +[reduce-memory.md](https://github.com/bitcoin/bitcoin/blob/master/doc/reduce-memory.md) +for further guidance on low-memory systems. (#34692) + +Mempool +------- + +The mempool has been reimplemented with a new design ("cluster mempool"), to +facilitate better decision-making when constructing block templates, evicting +transactions, relaying transactions, and validating replacement transactions +(RBF). Most changes should be transparent to users, but some behavior changes +are noted: + +- The mempool no longer enforces ancestor or descendant size/count limits. +Instead, two new default policy limits are introduced governing connected +components, or clusters, in the mempool, limiting clusters to 64 transactions +and up to 101 kB in virtual size. Transactions are considered to be in the same +cluster if they are connected to each other via any combination of parent/child +relationships in the mempool. These limits can be overridden using command-line +arguments; see the extended help (`-help-debug`) for more information. + +- Within the mempool, transactions are ordered based on the feerate at which +they are expected to be mined, which takes into account the full set, or +"chunk", of transactions that would be included together (e.g., a parent and its +child, or more complicated subsets of transactions). This ordering is utilized +by the algorithms that implement transaction selection for constructing block +templates; eviction from the mempool when it is full; and transaction relay +announcements to peers. + +- The replace-by-fee validation logic has been updated so that transaction +replacements are only accepted if the resulting mempool's feerate diagram is +strictly better than before the replacement. This eliminates all known cases of +replacements occurring that make the mempool worse off, which was possible under +previous RBF rules. For singleton transactions (that are in clusters by +themselves) it's sufficient for a replacement to have a higher fee and feerate +than the original. See [delvingbitcoin.org +post](https://delvingbitcoin.org/t/an-overview-of-the-cluster-mempool-proposal/393#rbf-can-now-be-made-incentive-compatible-for-miners-11) +for more information. + +- Two new RPCs have been added: `getmempoolcluster` will provide the set of +transactions in the same cluster as the given transaction, along with the +ordering of those transactions and grouping into chunks; and +`getmempoolfeeratediagram` will return the feerate diagram of the entire +mempool. + +- Chunk size and chunk fees are now also included in the output of +`getmempoolentry`. + +- The "CPFP Carveout" has been removed from the mempool logic. The CPFP carveout +allowed one additional child transaction to be added to a package that's already +at its descendant limit, but only if that child has exactly one ancestor (the +package's root) and is small (no larger than 10kvB). Nothing is allowed to +bypass the cluster count limit. It is expected that smart contracting use-cases +requiring similar functionality employ TRUC transactions and sibling eviction +instead going forward. + +- Some additional discussion can be found at +[doc/policy/mempool-terminology.md](https://github.com/bitcoin/bitcoin/blob/master/doc/policy/mempool-terminology.md) +and +[doc/policy/mempool-replacements.md](https://github.com/bitcoin/bitcoin/blob/master/doc/policy/mempool-replacements.md). + +P2P and network changes +----------------------- + +- Normally local transactions are broadcast to all connected peers with which we +do transaction relay. Now, for the `sendrawtransaction` RPC this behavior can be +changed to only do the broadcast via the Tor or I2P networks. A new boolean +option `-privatebroadcast` has been added to enable this behavior. This improves +the privacy of the transaction originator in two aspects: + 1. Their IP address (and thus geolocation) is never known to the recipients. + 2. If the originator sends two otherwise unrelated transactions, they will not + be linkable. This is because a separate connection is used for broadcasting + each transaction. (#29415) + +- New RPCs have been added to introspect and control private broadcast: +`getprivatebroadcastinfo` reports transactions currently being privately +broadcast, and `abortprivatebroadcast` removes matching transactions from the +private broadcast queue. (#34329) + +- Transactions participating in one-parent-one-child package relay can now have +the parent with a feerate lower than the `-minrelaytxfee` feerate, even 0 fee. +This expands the change from 28.0 to also cover packages of non-TRUC +transactions. Note that in general the package child can have additional +unconfirmed parents, but they must already be in-mempool for the new package to +be relayed. (#33892) + +- The release has asmap data embedded for the first time, allowing the asmap +feature to be used without any externally sourced file. The embedded map [was +created on 2026-03-05](https://github.com/bitcoin/bitcoin/pull/34696). Despite +the data being available, the option remains off-by-default. Users still need to +set `-asmap` or `-asmap=1` explicitly to make it possible to use a peer's ASN +(ISP/hoster identifier) in netgroup bucketing in order to ensure a higher +diversity in their peer set. + +Updated RPCs +------------ + +- `gettxspendingprevout` has 2 new optional arguments: `mempool_only` and +`return_spending_tx`. If `mempool_only` is true it will limit scans to the +mempool even if `txospenderindex` is available. If `return_spending_tx` is true, +the full spending tx will be returned. In addition if `txospenderindex` is +available and a confirmed spending transaction is found, its block hash will be +returned. (#24539) + +- The `getpeerinfo` RPC no longer returns the `startingheight` field unless the +configuration option `-deprecatedrpc=startingheight` is used. The +`startingheight` field will be fully removed in the next major release. (#34197) + +- The `getblock` RPC now returns a `coinbase_tx` object at verbosity levels 1, +2, and 3. It contains `version`, `locktime`, `sequence`, `coinbase` and +`witness`. This allows for efficiently querying coinbase transaction properties +without fetching the full transaction data at verbosity 2+. (#34512) + +REST API +-------- + +- A new REST API endpoint +(`/rest/blockpart/.?offset=&size=`) has been +introduced for efficiently fetching a range of bytes from block ``. +(#33657) + +Build System +------------ + +- The minimum supported Clang compiler version has been raised to 17.0 (#33555). +- The minimum supported GCC compiler version has been raised to 12.1 (#33842). + +Updated settings +---------------- + +- The `-paytxfee` startup option and the `settxfee` RPC are now deleted after +being deprecated in Bitcoin Core 30.0. They used to allow the user to set a +static fee rate for wallet transactions, which could potentially lead to +overpaying or underpaying. Users should instead rely on fee estimation or +specify a fee rate per transaction using the `fee_rate` argument in RPCs such as +`fundrawtransaction`, `sendtoaddress`, `send`, `sendall`, and `sendmany`. +(#32138) + +- Specifying `-asmap` or `-asmap=1` will load the embedded asmap data instead of +an external file. In previous releases, if `-asmap` was specified without a +filename, this would try to load an `ip_asn.map` data file. Now loading an +external asmap file always requires an explicit filename like +`-asmap=ip_asn.map`. + +- The `-maxorphantx` startup option has been removed. It was previously +deprecated and has no effect anymore since v30.0. (#33872) + +- `tor` has been removed as a network specification. It was deprecated in favour +of `onion` in v0.17.0. (#34031) + +- When `-logsourcelocations` is enabled, the log output now contains just the +function name instead of the entire function signature. (#34088) + +- The default `-dbcache` value has been increased to `1024` MiB from `450` MiB +on systems where at least `4096` MiB of RAM is detected. This is a performance +increase, but will use more memory. To maintain the previous behaviour, set +`-dbcache=450`. (#34692) + +- `-privatebroadcast` is added to enable private broadcast behavior for +`sendrawtransaction`. + +New settings +------------ + +- `-txospenderindex` enables the creation of a transaction output spender index +that, if present, will be scanned by `gettxspendingprevout` if a spending +transaction was not found in the mempool. (#24539) + +GUI changes +----------- + +- The GUI has been updated to Qt 6.8. (#34650) + +- The `createwallet`, `createwalletdescriptor` and `migratewallet` commands are +filtered from the console history to improve security and privacy. (gui#901) + +- The Restore Wallet dialog shows an error message if the restored wallet name +is empty. (gui#924) + +Fee Estimation +-------------- + +The Bitcoin Core fee estimator minimum fee rate bucket was updated from **1 +sat/vB** to **0.1 sat/vB**, which matches the node’s default `minrelaytxfee`. This +means that for a given confirmation target, if a sub-1 sat/vB fee rate bucket is +the minimum tracked with sufficient data, its average value will be returned as +the fee rate estimate. + +Restarting a node with this change invalidates previously saved +estimates in `fee_estimates.dat`, the fee estimator will start tracking fresh +stats. + +IPC Interface +------------- + +- The IPC mining interface now requires mining clients to use the latest +`mining.capnp` schema. Clients built against older schemas will fail when +calling `Init.makeMining` and receive an RPC error indicating the old mining +interface is no longer supported. Mining clients must update to the latest +schema and regenerate bindings to continue working. (#34568) +- `Mining.createNewBlock` now has a `cooldown` behavior (enabled by default) +that waits for IBD to finish and for the tip to catch up. This usually prevents +a flood of templates during startup, but is not guaranteed. (#34184) +- `Mining.interrupt()` can be used to interrupt `Mining.waitTipChanged` and +`Mining.createNewBlock`. (#34184) +- `Mining.createNewBlock` and `Mining.checkBlock` now require a `context` +parameter. +- `Mining.waitTipChanged` now has a default `timeout` (effectively infinite / +`maxDouble`) if the client omits it. +- `BlockTemplate.getCoinbaseTx()` now returns a structured `CoinbaseTx` instead +of raw bytes. +- Removed `BlockTemplate.getCoinbaseCommitment()` and +`BlockTemplate.getWitnessCommitmentIndex()`. +- Cap’n Proto default values were updated to match the corresponding C++ +defaults for mining-related option structs (e.g. `BlockCreateOptions`, +`BlockWaitOptions`, `BlockCheckOptions`). + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- Alexander Wiederin +- Alfonso Roman Zubeldia +- amisha +- ANAVHEOBA +- Andrew Toth +- Anthony Towns +- Antoine Poinsot +- ANtutov +- Anurag chavan +- Ava Chow +- bensig +- Ben Westgate +- billymcbip +- b-l-u-e +- Brandon Odiwuor +- brunoerg +- Bruno Garcia +- Calin Culianu +- Carl Dong +- Chandra Pratap +- Chris Stewart +- Coder +- Cory Fields +- da1sychain +- Daniela Brozzoni +- Daniel Pfeifer +- David Gumberg +- dergoegge +- Dmitry Goncharov +- Enoch Azariah +- Eugene Siegel +- Fabian Jahr +- fanquake +- Fibonacci747 +- flack +- frankomosh +- furszy +- glozow +- Greg Sanders +- Hao Xu +- Hennadii Stepanov +- Henry Romp +- Hodlinator +- ismaelsadeeq +- janb84 +- jayvaliya +- joaonevess +- John Moffett +- Josh Doman +- kevkevinpal +- l0rinc +- Luke Dashjr +- Mara van der Laan +- MarcoFalke +- marcofleon +- Martin Zumsande +- Matthew Zipkin +- Max Edwards +- Murch +- Musa Haruna +- naiyoma +- nervana21 +- Novo +- optout +- pablomartin4btc +- Padraic Slattery +- Pieter Wuille +- Pol Espinasa +- pythcoiner +- rkrux +- Robin David +- Roman Zeyde +- rustaceanrob +- Ryan Ofsky +- SatsAndSports +- scgbckbone +- Sebastian Falbesoner +- sedited +- seduless +- Sergi Delgado Segura +- Sjors Provoost +- SomberNight +- sstone +- stickies-v +- stratospher +- stringintech +- Suhas Daftuar +- tboy1337 +- TheCharlatan +- Tim Ruffing +- Vasil Dimov +- w0xlt +- WakeTrainDev +- Weixie Cui +- willcl-ark +- Woolfgm +- yancy +- Yash Bhutwala +- yuvicc +- zaidmstrr + +As well as to everyone that helped with translations on +[Transifex](https://explore.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 90ffd852e1a2..28bcfe83decc 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -25,6 +25,7 @@ Release Process - set `CLIENT_VERSION_MINOR` to `0` - set `CLIENT_VERSION_BUILD` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` +* Check with the security team whether there is any security advisory to pre-announce. #### Before branch-off @@ -296,7 +297,7 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes -- Announce the release: +- Announce the release, along with any security advisory pre-announcements: - bitcoin-dev and bitcoin-core-dev mailing list diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000000..d379dde1f501 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,18 @@ +[lint] +select = [ + "B006", # mutable-argument-default + "B008", # function-call-in-default-argument + "E", # pycodestyle errors + "F", # pyflakes errors + "PLE", # Pylint errors + "RUF001", # ambiguous-unicode-character-string + "RUF002", # ambiguous-unicode-character-docstring + "RUF003", # ambiguous-unicode-character-comment + "W", # pycodestyle warnings +] +ignore = [ + "E501", # line too long + "E712", # true-false comparison + "E731", # lambda assignment + "E741", # ambiguous-variable-name +] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index af40540715ee..524c28165cca 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -37,9 +37,15 @@ if(NOT CMAKE_ARCHIVE_OUTPUT_DIRECTORY) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) endif() +set(IS_SOURCE_TARBALL FALSE) +# git will expand the next line to "set(IS_SOURCE_TARBALL TRUE)" inside archives: +#$Format:%nset(IS_SOURCE_TARBALL TRUE)$ +if(NOT IS_SOURCE_TARBALL AND PROJECT_IS_TOP_LEVEL) + find_package(Git QUIET) +endif() add_custom_target(generate_build_info BYPRODUCTS ${PROJECT_BINARY_DIR}/src/bitcoin-build-info.h - COMMAND ${CMAKE_COMMAND} -DBUILD_INFO_HEADER_PATH=${PROJECT_BINARY_DIR}/src/bitcoin-build-info.h -DSOURCE_DIR=${PROJECT_SOURCE_DIR} -P ${PROJECT_SOURCE_DIR}/cmake/script/GenerateBuildInfo.cmake + COMMAND ${CMAKE_COMMAND} -DGIT_EXECUTABLE=${GIT_EXECUTABLE} -DBUILD_INFO_HEADER_PATH=${PROJECT_BINARY_DIR}/src/bitcoin-build-info.h -DSOURCE_DIR=${PROJECT_SOURCE_DIR} -P ${PROJECT_SOURCE_DIR}/cmake/script/GenerateBuildInfo.cmake COMMENT "Generating bitcoin-build-info.h" VERBATIM ) @@ -146,7 +152,6 @@ target_link_libraries(bitcoin_common bitcoin_util univalue secp256k1 - Boost::headers $ $<$:ws2_32> ) @@ -169,7 +174,6 @@ if(ENABLE_WALLET) bitcoin_wallet bitcoin_common bitcoin_util - Boost::headers ) install_binary_component(bitcoin-wallet HAS_MANPAGE) endif() @@ -228,6 +232,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/mempool_persist.cpp node/mempool_persist_args.cpp node/miner.cpp + node/mining_args.cpp node/mini_miner.cpp node/minisketchwrapper.cpp node/peerman_args.cpp @@ -359,8 +364,6 @@ if(BUILD_CLI) bitcoin_common bitcoin_ipc bitcoin_util - libevent::core - libevent::extra ) install_binary_component(bitcoin-cli HAS_MANPAGE) endif() diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 5db632c7e1ed..14baf1a643d3 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -24,6 +23,7 @@ #include #include #include +#include #include #include diff --git a/src/addrman.cpp b/src/addrman.cpp index d3dae59ae790..92178f30089b 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include #include #include diff --git a/src/addrman_impl.h b/src/addrman_impl.h index c1716f891d62..88ee92e1b04e 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -5,13 +5,13 @@ #ifndef BITCOIN_ADDRMAN_IMPL_H #define BITCOIN_ADDRMAN_IMPL_H -#include #include #include #include #include #include #include +#include #include #include diff --git a/src/banman.cpp b/src/banman.cpp index 3c1f64e2263c..54789f8d7d47 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -6,10 +6,10 @@ #include #include -#include #include #include #include +#include #include #include diff --git a/src/base58.cpp b/src/base58.cpp index 19a5bd3e550f..ad5880f345d6 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -37,25 +37,25 @@ static const int8_t mapBase58[256] = { -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, }; -[[nodiscard]] static bool DecodeBase58(const char* psz, std::vector& vch, int max_ret_len) +[[nodiscard]] static bool DecodeBase58(const char* psz, const char* end, std::vector& vch, int max_ret_len) { // Skip leading spaces. - while (*psz && IsSpace(*psz)) + while (psz != end && IsSpace(*psz)) psz++; // Skip and count leading '1's. int zeroes = 0; int length = 0; - while (*psz == '1') { + while (psz != end && *psz == '1') { zeroes++; if (zeroes > max_ret_len) return false; psz++; } // Allocate enough space in big-endian base256 representation. - int size = strlen(psz) * 733 /1000 + 1; // log(58) / log(256), rounded up. + int size = (end - psz) * 733 / 1000 + 1; // log(58) / log(256), rounded up. std::vector b256(size); // Process the characters. static_assert(std::size(mapBase58) == 256, "mapBase58.size() should be 256"); // guarantee not out of range - while (*psz && !IsSpace(*psz)) { + while (psz != end && !IsSpace(*psz)) { // Decode base58 character int carry = mapBase58[(uint8_t)*psz]; if (carry == -1) // Invalid b58 character @@ -72,9 +72,9 @@ static const int8_t mapBase58[256] = { psz++; } // Skip trailing spaces. - while (IsSpace(*psz)) + while (psz != end && IsSpace(*psz)) psz++; - if (*psz != 0) + if (psz != end) return false; // Skip leading zeroes in b256. std::vector::iterator it = b256.begin() + (size - length); @@ -126,12 +126,12 @@ std::string EncodeBase58(std::span input) return str; } -bool DecodeBase58(const std::string& str, std::vector& vchRet, int max_ret_len) +bool DecodeBase58(std::string_view str, std::vector& vchRet, int max_ret_len) { if (!ContainsNoNUL(str)) { return false; } - return DecodeBase58(str.c_str(), vchRet, max_ret_len); + return DecodeBase58(str.data(), str.data() + str.size(), vchRet, max_ret_len); } std::string EncodeBase58Check(std::span input) @@ -143,9 +143,9 @@ std::string EncodeBase58Check(std::span input) return EncodeBase58(vch); } -[[nodiscard]] static bool DecodeBase58Check(const char* psz, std::vector& vchRet, int max_ret_len) +[[nodiscard]] static bool DecodeBase58Check(const char* psz, const char* end, std::vector& vchRet, int max_ret_len) { - if (!DecodeBase58(psz, vchRet, max_ret_len > std::numeric_limits::max() - 4 ? std::numeric_limits::max() : max_ret_len + 4) || + if (!DecodeBase58(psz, end, vchRet, max_ret_len > std::numeric_limits::max() - 4 ? std::numeric_limits::max() : max_ret_len + 4) || (vchRet.size() < 4)) { vchRet.clear(); return false; @@ -160,10 +160,10 @@ std::string EncodeBase58Check(std::span input) return true; } -bool DecodeBase58Check(const std::string& str, std::vector& vchRet, int max_ret) +bool DecodeBase58Check(std::string_view str, std::vector& vchRet, int max_ret) { if (!ContainsNoNUL(str)) { return false; } - return DecodeBase58Check(str.c_str(), vchRet, max_ret); + return DecodeBase58Check(str.data(), str.data() + str.size(), vchRet, max_ret); } diff --git a/src/base58.h b/src/base58.h index f258163bc3da..fa227990fc9c 100644 --- a/src/base58.h +++ b/src/base58.h @@ -17,6 +17,7 @@ #include #include +#include #include /** @@ -28,7 +29,7 @@ std::string EncodeBase58(std::span input); * Decode a base58-encoded string (str) into a byte vector (vchRet). * return true if decoding is successful. */ -[[nodiscard]] bool DecodeBase58(const std::string& str, std::vector& vchRet, int max_ret_len); +[[nodiscard]] bool DecodeBase58(std::string_view str, std::vector& vchRet, int max_ret_len); /** * Encode a byte span into a base58-encoded string, including checksum @@ -39,6 +40,6 @@ std::string EncodeBase58Check(std::span input); * Decode a base58-encoded string (str) that includes a checksum into a byte * vector (vchRet), return true if decoding is successful */ -[[nodiscard]] bool DecodeBase58Check(const std::string& str, std::vector& vchRet, int max_ret_len); +[[nodiscard]] bool DecodeBase58Check(std::string_view str, std::vector& vchRet, int max_ret_len); #endif // BITCOIN_BASE58_H diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt index e1d5e4097ff4..3c81e7986da9 100644 --- a/src/bench/CMakeLists.txt +++ b/src/bench/CMakeLists.txt @@ -70,6 +70,10 @@ target_link_libraries(bench_bitcoin Boost::headers ) +if(WITH_EMBEDDED_ASMAP) + target_sources(bench_bitcoin PRIVATE asmap.cpp) +endif() + if(ENABLE_WALLET) target_sources(bench_bitcoin PRIVATE diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index fc081d9ff4b0..703b4d245db4 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -162,8 +162,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) CreateAddresses(); std::optional addrman; - bench.epochIterations(1) - .setup([&] { + bench.setup([&] { addrman.emplace(EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO); AddAddressesToAddrMan(*addrman); }) diff --git a/src/bench/asmap.cpp b/src/bench/asmap.cpp new file mode 100644 index 000000000000..e834ea04a278 --- /dev/null +++ b/src/bench/asmap.cpp @@ -0,0 +1,161 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +static void BenchGetMappedAS(benchmark::Bench& bench, std::span addrs, bool check = true) +{ + std::span asmap{node::data::ip_asn}; + assert(!asmap.empty() && CheckStandardAsmap(asmap)); + auto netgroupman{NetGroupManager::WithEmbeddedAsmap(asmap)}; + + bench.batch(addrs.size()).run([&] { + for (const CNetAddr& addr : addrs) { + // The embedded ASMap file might change over time and cause some + // addresses to become unmapped. We check the mapping to ensure + // the addresses are actually mapped. + assert((netgroupman.GetMappedAS(addr) > 0) == check); + } + }); +} + +static CNetAddr LookupAddr(const std::string& address) { return LookupHost(address, /*fAllowLookup=*/false).value(); } + +static void ASMapGetMappedASQuad9v4(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("9.9.9.9")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASQuad9v6(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("2620:fe::fe")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASCloudflarev4(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("1.1.1.1")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASCloudflarev6(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("2606:4700:4700::1111")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASGooglev4(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("8.8.8.8")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASGooglev6(benchmark::Bench& bench) +{ + std::vector addrs{LookupAddr("2001:4860:4860::8888")}; + BenchGetMappedAS(bench, addrs); +} + +static void ASMapGetMappedASUnmappedv4(benchmark::Bench& bench) +{ + // Reserved as per RFC 5737 (unmapped) + std::vector addrs{LookupAddr("203.0.113.0")}; + BenchGetMappedAS(bench, addrs, /*check=*/false); +} + +static void ASMapGetMappedASUnmappedv6(benchmark::Bench& bench) +{ + // Reserved as per RFC 9637 (unmapped) + std::vector addrs{LookupAddr("3fff::1")}; + BenchGetMappedAS(bench, addrs, /*check=*/false); +} + +static void ASMapGetMappedASMulti(benchmark::Bench& bench) +{ + // A list of 25 IPv4 and 25 IPv6 addresses randomly sampled across + // 50 ASNs to have a mix of different lookup times. These have been + // masked to /24 and /64 respectively. Some of these could become + // unmapped when updating the embedded ASMap file, which will cause + // the benchmarks to assert and the IPs will need to be changed out. + std::array addrs{ + LookupAddr("5.8.10.1"), + LookupAddr("14.53.20.1"), + LookupAddr("34.86.160.1"), + LookupAddr("67.68.205.1"), + LookupAddr("68.163.58.1"), + LookupAddr("69.138.189.1"), + LookupAddr("72.211.58.1"), + LookupAddr("73.124.158.1"), + LookupAddr("77.179.43.1"), + LookupAddr("80.79.125.1"), + LookupAddr("81.217.170.1"), + LookupAddr("82.216.149.1"), + LookupAddr("84.52.201.1"), + LookupAddr("87.184.174.1"), + LookupAddr("90.221.151.1"), + LookupAddr("95.31.136.1"), + LookupAddr("99.234.174.1"), + LookupAddr("105.98.199.1"), + LookupAddr("146.190.174.1"), + LookupAddr("154.16.157.1"), + LookupAddr("172.81.183.1"), + LookupAddr("173.24.74.1"), + LookupAddr("195.99.226.1"), + LookupAddr("213.197.14.1"), + LookupAddr("220.255.248.1"), + + LookupAddr("2001:16a2:c0b0:58b7::1"), + LookupAddr("2001:67c:e60:c0c::1"), + LookupAddr("2001:99a:213:27f0::1"), + LookupAddr("2001:9e8:894b:be00::1"), + LookupAddr("2406:2d40:1ebc:3508::1"), + LookupAddr("2600:1015:a020:1e00::1"), + LookupAddr("2600:1700:4228:a800::1"), + LookupAddr("2601:603:5000:3309::1"), + LookupAddr("2601:cd:ce01:9610::1"), + LookupAddr("2603:800c:25f0:8350::1"), + LookupAddr("2604:3d09:f89:d100::1"), + LookupAddr("2607:fb90:236f:821f::1"), + LookupAddr("2800:2331:5440:ba3::1"), + LookupAddr("2a00:16e0:1012:c108::1"), + LookupAddr("2a00:23c6:9d44:7801::1"), + LookupAddr("2a00:79c0:609:4900::1"), + LookupAddr("2a01:4f8:13a:1f8d::1"), + LookupAddr("2a01:e0a:8a3:ab90::1"), + LookupAddr("2a02:810b:449e:c900::1"), + LookupAddr("2a02:8308:20d:e800::1"), + LookupAddr("2a0a:ef40:331:4401::1"), + LookupAddr("2a0c:5a86:a002:e700::1"), + LookupAddr("2a10:3781:1d7:1::1"), + LookupAddr("2a12:26c0:3303:a100::1"), + LookupAddr("2a12:a800:2:1::1"), + }; + + FastRandomContext rng{/*fDeterministic=*/true}; + std::ranges::shuffle(addrs, rng); + + BenchGetMappedAS(bench, addrs, /*check=*/true); +} + +BENCHMARK(ASMapGetMappedASQuad9v4); +BENCHMARK(ASMapGetMappedASQuad9v6); +BENCHMARK(ASMapGetMappedASCloudflarev4); +BENCHMARK(ASMapGetMappedASCloudflarev6); +BENCHMARK(ASMapGetMappedASGooglev4); +BENCHMARK(ASMapGetMappedASGooglev6); +BENCHMARK(ASMapGetMappedASUnmappedv4); +BENCHMARK(ASMapGetMappedASUnmappedv6); +BENCHMARK(ASMapGetMappedASMulti); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 702f2c093668..be03917417e6 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include