CI: consolidate workflows for source and docs check

A bunch of tiny jobs that run various source or documentation checks are
consolidated into two workflow files: checksrc.yml and checkdocs.yml.
This reduces the proliferation of new files containing one-line checks
and brings those that operate similarly together for better reasoning
about them. The man-examples check is also now running again for the
first time in 7 months.

Various calls to find, xargs and git ls-files are changed where possible
to use NUL line terminators in pipes to avoid issues with oddly-named
files that might find their way into the repo.

Closes #14654
This commit is contained in:
Dan Fandrich 2024-08-22 21:24:03 -07:00
parent 6429ce8e5f
commit 5629bb7cf6
14 changed files with 219 additions and 435 deletions

View file

@ -1,42 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: badwords
on:
# Trigger the workflow on push or pull requests, but only for the
# master branch
push:
branches:
- master
- '*/ci'
pull_request:
branches:
- master
permissions: {}
jobs:
docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: check
run: ./.github/scripts/badwords.pl < .github/scripts/badwords.txt docs/*.md docs/libcurl/*.md docs/libcurl/opts/*.md docs/cmdline-opts/*.md docs/TODO docs/KNOWN_BUGS tests/*.md
source:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
# we allow some extra in source code
- name: trim wordlist
run: grep -Ev '(\\bwill| url | dir )' .github/scripts/badwords.txt > .github/scripts/source.txt
- name: check
run: ./.github/scripts/badwords.pl < .github/scripts/source.txt `git ls-files -- src lib include`

View file

@ -1,23 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: bincheck
on:
push:
branches:
- master
pull_request:
branches:
- master
permissions: {}
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: Check for binary files
run: ./.github/scripts/binarycheck.pl

146
.github/workflows/checkdocs.yml vendored Normal file
View file

@ -0,0 +1,146 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
# This workflow contains tests that operate on documentation files only. Some
# checks modify the source so they cannot be combined into a single job.
name: Docs
on:
push:
branches:
- master
- '*/ci'
paths:
- '.github/workflows/checkdocs.yml'
- '.github/scripts/**'
- '**.md'
- 'docs/*'
pull_request:
branches:
- master
paths:
- '.github/workflows/checkdocs.yml'
- '.github/scripts/**'
- '**.md'
- 'docs/*'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
proselint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: install prereqs
run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
sudo apt-get install python3-proselint
# config file help: https://github.com/amperser/proselint/
- name: create proselint config
run: |
cat <<JSON > $HOME/.proselintrc
{
"checks": {
"typography.diacritical_marks": false,
"typography.symbols": false,
"annotations.misc": false,
"security.password": false
}
}
JSON
- name: trim headers off all *.md files
run: git ls-files -z '*.md' | xargs -0 -n1 .github/scripts/trimmarkdownheader.pl
- name: check prose
run: git ls-files -z '*.md' | grep -Evz 'CHECKSRC.md|DISTROS.md|CURLOPT_INTERFACE.md|interface.md' | xargs -0 proselint README
# This is for CHECKSRC and files with aggressive exclamation mark needs
- name: create second proselint config
run: |
cat <<JSON > $HOME/.proselintrc
{
"checks": {
"typography.diacritical_marks": false,
"typography.symbols": false,
"typography.exclamation": false,
"annotations.misc": false
}
}
JSON
- name: check special prose
run: proselint docs/CHECKSRC.md docs/libcurl/opts/CURLOPT_INTERFACE.md docs/cmdline-opts/interface.md
# Docs: https://github.com/marketplace/actions/markdown-link-check
linkcheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: trim the cmdline docs markdown files
run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl
- uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1
with:
use-quiet-mode: 'yes'
spellcheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: trim all man page *.md files
run: find docs -name "*.md" ! -name "_*" -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl
- name: trim libcurl man page *.md files
run: find docs/libcurl \( -name "curl_*.md" -o -name "libcurl*.md" \) -print0 | xargs -0 -n1 .github/scripts/cleanspell.pl
- name: trim libcurl option man page *.md files
run: find docs/libcurl/opts -name "CURL*.md" -print0 | xargs -0 -n1 .github/scripts/cleanspell.pl
- name: trim cmdline docs markdown _*.md files
run: find docs/cmdline-opts -name "_*.md" -print0 | xargs -0 -n1 .github/scripts/cleancmd.pl --no-header
- name: setup the custom wordlist
run: grep -v '^#' .github/scripts/spellcheck.words > wordlist.txt
- name: Check Spelling
uses: rojopolis/spellcheck-github-actions@a0fba0ca8b9e552d4241ea5ccfaa4ca4162622d0 # v0
with:
config_path: .github/scripts/spellcheck.yaml
badwords-synopsis:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: badwords
run: .github/scripts/badwords.pl < .github/scripts/badwords.txt docs/*.md docs/libcurl/*.md docs/libcurl/opts/*.md docs/cmdline-opts/*.md docs/TODO docs/KNOWN_BUGS tests/*.md
- name: verify-synopsis
run: .github/scripts/verify-synopsis.pl docs/libcurl/curl*.md
man-examples:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: render nroff versions
run: autoreconf -fi && ./configure --without-ssl --without-libpsl && make -C docs
- name: verify examples
run: .github/scripts/verify-examples.pl docs/libcurl/curl*.3 docs/libcurl/opts/*.3

View file

@ -2,28 +2,93 @@
#
# SPDX-License-Identifier: curl
name: checksrc
# This workflow contains checks at the source code level only.
name: Source
on:
# Trigger the workflow on push or pull requests, but only for the
# master branch
push:
branches:
- master
- '*/ci'
paths-ignore:
- '**/*.md'
- '**/CMakeLists.txt'
- '.azure-pipelines.yml'
- '.circleci/**'
- 'appveyor.*'
- 'CMake/**'
- 'plan9/**'
- 'tests/data/**'
- 'winbuild/**'
pull_request:
branches:
- master
paths-ignore:
- '**/*.md'
- '**/CMakeLists.txt'
- '.azure-pipelines.yml'
- '.circleci/**'
- 'appveyor.*'
- 'CMake/**'
- 'plan9/**'
- 'tests/data/**'
- 'winbuild/**'
permissions: {}
jobs:
all:
checksrc:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: check
run: git ls-files "*.[ch]" | xargs -n1 ./scripts/checksrc.pl
run: git ls-files -z "*.[ch]" | xargs -0 -n1 ./scripts/checksrc.pl
codespell:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: install
run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
sudo apt-get update
sudo apt-get install codespell
- name: spellcheck
run: codespell --skip src/tool_hugehelp.c -I .github/scripts/codespell-ignore.txt include src lib
reuse:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: REUSE Compliance Check
uses: fsfe/reuse-action@3ae3c6bdf1257ab19397fab11fd3312144692083 # v4
miscchecks:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: shellcheck
run: .github/scripts/shellcheck.sh
- name: spacecheck
run: .github/scripts/spacecheck.pl
- name: binarycheck
run: .github/scripts/binarycheck.pl
# we allow some extra in source code
- name: badwords
run: >
grep -Ev '(\\bwill| url | dir )' .github/scripts/badwords.txt |
.github/scripts/badwords.pl `git ls-files -- src lib include`

View file

@ -1,39 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: Codespell
on:
push:
branches:
- master
- '*/ci'
paths:
- 'lib/**'
- 'src/**'
- 'include/**'
pull_request:
branches:
- master
- 'lib/**'
- 'src/**'
- 'include/**'
permissions: {}
jobs:
codespell:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: install
run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
sudo apt-get update
sudo apt-get install codespell
- name: spellcheck
run: codespell --skip src/tool_hugehelp.c -I .github/scripts/codespell-ignore.txt include src lib

View file

@ -1,41 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: Markdown links
on:
push:
branches:
- master
- '*/ci'
paths:
- '.github/workflows/linkcheck.yml'
- '**.md'
pull_request:
branches:
- master
paths:
- '.github/workflows/linkcheck.yml'
- '**.md'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
# Docs: https://github.com/marketplace/actions/markdown-link-check
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
name: checkout
- name: trim the cmdline docs markdown files
run: find docs/cmdline-opts -name "*.md" ! -name "_*" ! -name MANPAGE.md | xargs -n1 ./.github/scripts/cleancmd.pl
- uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1
with:
use-quiet-mode: 'yes'

View file

@ -1,37 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: manpage examples
on:
push:
branches:
- master
- '*/ci'
paths:
- 'docs/libcurl/curl_*.3'
- 'docs/libcurl/opts/*.3'
- '.github/scripts/verify-examples.pl'
pull_request:
branches:
- master
paths:
- 'docs/libcurl/curl_*.3'
- 'docs/libcurl/opts/*.3'
- '.github/scripts/verify-examples.pl'
permissions: {}
jobs:
verify:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: render nroff versions
run: autoreconf -fi && ./configure --without-ssl --without-libpsl && make -C docs
- name: verify examples
run: ./.github/scripts/verify-examples.pl docs/libcurl/curl*.3 docs/libcurl/opts/*.3

View file

@ -1,74 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: proselint
on:
push:
branches:
- master
- '*/ci'
paths:
- '.github/workflows/proselint.yml'
- '**.md'
pull_request:
branches:
- master
paths:
- '.github/workflows/proselint.yml'
- '**.md'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: install prereqs
run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
sudo apt-get install python3-proselint
# config file help: https://github.com/amperser/proselint/
- name: create proselint config
run: |
cat <<JSON > $HOME/.proselintrc
{
"checks": {
"typography.diacritical_marks": false,
"typography.symbols": false,
"annotations.misc": false,
"security.password": false
}
}
JSON
- name: trim headers off all *.md files
run: git ls-files '*.md' | xargs -n1 ./.github/scripts/trimmarkdownheader.pl
- name: check prose
run: git ls-files '*.md' | grep -Ev 'CHECKSRC.md|DISTROS.md|CURLOPT_INTERFACE.md|interface.md' | xargs proselint README
# This is for CHECKSRC and files with aggressive exclamation mark needs
- name: create second proselint config
run: |
cat <<JSON > $HOME/.proselintrc
{
"checks": {
"typography.diacritical_marks": false,
"typography.symbols": false,
"typography.exclamation": false,
"annotations.misc": false
}
}
JSON
- name: check special prose
run: proselint docs/CHECKSRC.md docs/libcurl/opts/CURLOPT_INTERFACE.md docs/cmdline-opts/interface.md

View file

@ -1,29 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
# SPDX-FileCopyrightText: 2022 Free Software Foundation Europe e.V. <https://fsfe.org>
#
# SPDX-License-Identifier: curl
name: REUSE compliance
on:
push:
branches:
- master
- '*/ci'
pull_request:
branches:
- master
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: REUSE Compliance Check
uses: fsfe/reuse-action@3ae3c6bdf1257ab19397fab11fd3312144692083 # v4

View file

@ -1,28 +0,0 @@
# Copyright (C) Viktor Szakats
#
# SPDX-License-Identifier: curl
name: shellcheck
on:
push:
branches:
- master
pull_request:
branches:
- master
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
shellcheck:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: 'shellcheck'
run: .github/scripts/shellcheck.sh

View file

@ -1,28 +0,0 @@
# Copyright (C) Viktor Szakats
#
# SPDX-License-Identifier: curl
name: spacecheck
on:
push:
branches:
- master
pull_request:
branches:
- master
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
permissions: {}
jobs:
spacecheck:
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: 'spacecheck'
run: .github/scripts/spacecheck.pl

View file

@ -1,50 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: spell
on:
push:
branches:
- master
paths:
- '**.md'
- '**/spellcheck.yml'
- '**/spellcheck.yaml'
- '.github/scripts/*'
pull_request:
branches:
- master
paths:
- '**.md'
- '**/spellcheck.yml'
- '**/spellcheck.yaml'
- '.github/scripts/*'
permissions: {}
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: trim all man page *.md files
run: find docs -name "*.md" ! -name "_*" | xargs -n1 ./.github/scripts/cleancmd.pl
- name: trim libcurl man page *.md files
run: find docs/libcurl -name "curl_*.md" -o -name "libcurl*.md" | xargs -n1 ./.github/scripts/cleanspell.pl
- name: trim libcurl option man page *.md files
run: find docs/libcurl/opts -name "CURL*.md" | xargs -n1 ./.github/scripts/cleanspell.pl
- name: trim cmdline docs markdown _*.md files
run: find docs/cmdline-opts -name "_*.md" | xargs -n1 ./.github/scripts/cleancmd.pl --no-header
- name: setup the custom wordlist
run: grep -v '^#' .github/scripts/spellcheck.words > wordlist.txt
- name: Check Spelling
uses: rojopolis/spellcheck-github-actions@a0fba0ca8b9e552d4241ea5ccfaa4ca4162622d0 # v0
with:
config_path: .github/scripts/spellcheck.yaml

View file

@ -1,34 +0,0 @@
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# SPDX-License-Identifier: curl
name: SYNOPSIS
on:
push:
branches:
- master
- '*/ci'
paths:
- 'docs/libcurl/curl_*.md'
- 'scripts/*.pl'
- '.github/workflows/*.yml'
pull_request:
branches:
- master
paths:
- 'docs/libcurl/curl_*.md'
- 'scripts/*.pl'
- '.github/workflows/*.yml'
permissions: {}
jobs:
verify:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: verify-synopsis
run: ./.github/scripts/verify-synopsis.pl docs/libcurl/curl*.md