From dfae747e84ff26d7bcc4a383c0800d922957d8a7 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 26 Feb 2021 11:48:28 +0200 Subject: [PATCH 1/4] scripts/update-gh-pages: manage a Helm repo Make the update-gh-pages.sh script to maintain a Helm charts repository under charts/ subdirectory in gh-pages. The script now (always) scans throught all release assets and injects any found Helm chart archives into the Helm repo. In practice, new assets in all Github releases are scanned and the Helm repo is updated on any update of the master or release branches or on any new tags. Asset ids are tracked/cached in order to avoid unnecessary downloads, but also, to capture any changes in assets that were already merged in the repo index. After this a user is able to do something like $ helm repo add nfd http://kubernetes-sigs.github.io/node-feature-discovery/charts ... $ helm repo update ... $ helm install nfd/node-feature-discovery --namespace nfd --create-namespace --generate-name ... --- .github/workflows/gh-pages.yml | 5 +++ scripts/github/update-gh-pages.sh | 62 +++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index c500d7a87..25ba0d543 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -12,6 +12,11 @@ jobs: runs-on: ubuntu-latest steps: + - name: Install dependencies + run: | + sudo apt-get install -y jq curl + curl -sfL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash -s -- --version v3.5.2 + - name: Check out repo uses: actions/checkout@v1 with: diff --git a/scripts/github/update-gh-pages.sh b/scripts/github/update-gh-pages.sh index 3456fec33..d7be9e2e7 100755 --- a/scripts/github/update-gh-pages.sh +++ b/scripts/github/update-gh-pages.sh @@ -22,11 +22,59 @@ create_versions_js() { # 'stable' is a symlink pointing to the latest version [ -f stable ] && echo " { name: 'stable', url: '$_baseurl/stable' }," for f in `ls -d */ | tr -d /` ; do - echo " { name: '$f', url: '$_baseurl/$f' }," + if [ -f "$f/index.html" ]; then + echo " { name: '$f', url: '$_baseurl/$f' }," + fi done echo -e " ];\n}" } +# Helper for updating help repo index +update_helm_repo_index() { + echo "Updating Helm repo index" + + # TODO: with a lot of releases github API will paginate and this will break + releases="`curl -sSf -H 'Accept: application/vnd.github.v3+json' \ + $GITHUB_API_URL/repos/$GITHUB_REPOSITORY/releases | jq -c '.[]'`" + + for release_meta in $releases; do + # Set fields we're interested in as shell variables + eval `echo $release_meta | jq -r '{tag_name, url, assets} | keys[] as $k | "\($k)='"'"'\(.[$k])'"'"'"'` + + for asset_meta in `echo $assets | jq -c '.[]'`; do + # Set fields we're interested in as "asset_" shell variables + eval `echo $asset_meta | jq -r '{id, name, url, browser_download_url} | keys[] as $k | "local asset_\($k)=\(.[$k])"'` + + if [[ "$asset_name" != node-feature-discovery-chart-*tgz ]]; then + echo "Asset $asset_name does not look like a Helm chart archive, skipping..." + continue + fi + + # Check if the asset has changed + asset_id_old=`cat "$asset_name".id 2> /dev/null || :` + if [[ $asset_id_old == $asset_id ]]; then + echo "$asset_name (id=$asset_id) unchanged, skipping..." + continue + fi + + # Update helm repo index + local tmpdir="`mktemp -d`" + + echo "Downloading $asset_name..." + curl -sSfL -H "Accept:application/octet-stream" -o "$tmpdir/$asset_name" $asset_url + + echo "Updating helm index for $asset_name..." + local download_baseurl=`dirname $asset_browser_download_url` + helm repo index "$tmpdir" --merge index.yaml --url $download_baseurl + cp "$tmpdir/index.yaml" . + rm -rf "$tmpdir" + + # Update id cache file + echo $asset_id > "$asset_name".id + done + done +} + # # Argument parsing # @@ -132,7 +180,7 @@ else fi # Switch to work in the gh-pages worktree -cd "$build_dir" +pushd "$build_dir" _stable=`(ls -d1 v*/ || :) | sort -n | tail -n1` [ -n "$_stable" ] && ln -sfT "$_stable" stable @@ -140,10 +188,18 @@ _stable=`(ls -d1 v*/ || :) | sort -n | tail -n1` # Detect existing versions from the gh-pages branch create_versions_js > versions.js +# Create index.html cat > index.html << EOF EOF +# Update Helm repo +mkdir -p charts +pushd charts +update_helm_repo_index +popd + +# Check if there were any changes in the repo if [ -z "`git status --short`" ]; then echo "No new content, gh-pages branch already up-to-date" exit 0 @@ -156,7 +212,7 @@ echo "Committing changes..." git add . git commit $amend -m "$commit_msg" -cd - +popd echo "gh-pages branch successfully updated" From f80e683e3e17f57f5a0163112c52911f0fcedfec Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 26 Feb 2021 15:21:31 +0200 Subject: [PATCH 2/4] scripts/prepare-release: package Helm chart Prepare a signed Helm chart package to be uploaded to the Github release page. --- scripts/prepare-release.sh | 50 +++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/scripts/prepare-release.sh b/scripts/prepare-release.sh index cc9b52a63..61fe77469 100755 --- a/scripts/prepare-release.sh +++ b/scripts/prepare-release.sh @@ -5,10 +5,23 @@ this=`basename $0` usage () { cat << EOF -Usage: $this [-h] RELEASE_VERSION +Usage: $this [-h] RELEASE_VERSION GPG_KEY GPG_KEYRING Options: -h show this help and exit + +Example: + + $this v0.1.2 "Jane Doe " ~/.gnupg/secring.gpg + + +NOTE: The GPG key should be associated with the signer's Github account. + +NOTE: Helm is not compatible with GnuPG v2 and you need to export the secret + keys in order for Helm to be able to sign the package: + + gpg --export-secret-keys > ~/.gnupg/secring.gpg + EOF } @@ -28,13 +41,21 @@ done shift "$((OPTIND - 1))" # Check that no extra args were provided -if [ $# -gt 1 ]; then - echo -e "ERROR: unknown arguments: $@\n" +if [ $# -ne 3 ]; then + if [ $# -lt 3 ]; then + echo -e "ERROR: too few arguments\n" + else + echo -e "ERROR: unknown arguments: ${@:4}\n" + fi usage exit 1 fi release=$1 +key="$2" +keyring="$3" +shift 3 + container_image=k8s.gcr.io/nfd/node-feature-discovery:$release # @@ -48,6 +69,7 @@ fi if [[ $release =~ ^(v[0-9]+\.[0-9]+)(\..+)?$ ]]; then docs_version=${BASH_REMATCH[1]} + semver=${release:1} else echo -e "ERROR: invalid RELEASE_VERSION '$release'" exit 1 @@ -81,3 +103,25 @@ echo Patching test/e2e/node_feature_discovery.go flag defaults to k8s.gcr.io/nfd sed -e s'!"nfd\.repo",.*,!"nfd.repo", "k8s.gcr.io/nfd/node-feature-discovery",!' \ -e s"!\"nfd\.tag\",.*,!\"nfd.tag\", \"$release\",!" \ -i test/e2e/node_feature_discovery.go + +# +# Create release assets to be uploaded +# +helm package deployment/node-feature-discovery/ --version $semver --sign \ + --key "$key" --keyring "$keyring" + +chart_name="node-feature-discovery-chart-$semver.tgz" +mv node-feature-discovery-$semver.tgz $chart_name +mv node-feature-discovery-$semver.tgz.prov $chart_name.prov + +cat << EOF + +******************************************************************************* +*** Please manually upload the following generated files to the Github release +*** page: +*** +*** $chart_name +*** $chart_name.prov +*** +******************************************************************************* +EOF From 8cf93677989a92b17159e5554f660c1fbeb80461 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 26 Feb 2021 16:08:19 +0200 Subject: [PATCH 3/4] github: update release process in new-release issue template Capture process for Helm charts and remove some outdated bits. --- .github/ISSUE_TEMPLATE/new-release.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/new-release.md b/.github/ISSUE_TEMPLATE/new-release.md index 9f5bc37d7..2dbb8a4d9 100644 --- a/.github/ISSUE_TEMPLATE/new-release.md +++ b/.github/ISSUE_TEMPLATE/new-release.md @@ -19,10 +19,13 @@ Please do not remove items from the checklist `git push release-0.$MAJ` - [ ] Run `scripts/prepare-release.sh $VERSION` to turn references to point to the upcoming release (README, deployment templates, docs configuration, test/e2e flags), submit a PR agains the release branch + - An OWNER prepares a draft release + - [ ] Create a draft release at [Github releases page](https://github.com/kubernetes-sigs/node-feature-discovery/releases) + - [ ] Write the change log into the draft release + - [ ] Upload release artefacts generated by `prepare-release.sh` script above to the draft release - [ ] An OWNER runs `git tag -s $VERSION` and inserts the changelog into the tag description. -discovery/0.$MAJ/ - [ ] An OWNER pushes the tag with `git push $VERSION` - Triggers prow to build and publish a staging container image @@ -31,9 +34,8 @@ discovery/0.$MAJ/ https://kubernetes-sigs.github.io/node-feature - [ ] Submit a PR against [k8s.io](https://github.com/kubernetes/k8s.io), updating `k8s.gcr.io/images/k8s-staging-nfd/images.yaml` to promote the container image to production - [ ] Wait for the PR to be merged and verify that the image (`k8s.gcr.io/nfd/node-feature-discovery:$VERSION`) is available. -- [ ] Write the change log into the [Github release info](https://github.com/kubernetes-sigs/node-feature-discovery/releases). +- [ ] Publish the draft release prepared at the [Github releases page](https://github.com/kubernetes-sigs/node-feature-discovery/releases) - [ ] Add a link to the tagged release in this issue. -- [ ] Update `index.html` in `gh-pages` branch to point to the latest release - [ ] Send an announcement email to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] node-feature-discovery $VERSION is released` - [ ] Add a link to the release announcement in this issue - [ ] Close this issue From fd2dcc227584ed89953ceddebe7e7603622017ef Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Tue, 16 Mar 2021 21:48:14 +0200 Subject: [PATCH 4/4] scripts/prepare-release: use gpg for signing Helm chart Need to create the provenance file by hand as we mangle the name of the chart archive. However, this also provides better user experience (for the release manager) as gpg version 2.1 and later are supported. --- scripts/prepare-release.sh | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/scripts/prepare-release.sh b/scripts/prepare-release.sh index 61fe77469..f231c5eae 100755 --- a/scripts/prepare-release.sh +++ b/scripts/prepare-release.sh @@ -5,26 +5,31 @@ this=`basename $0` usage () { cat << EOF -Usage: $this [-h] RELEASE_VERSION GPG_KEY GPG_KEYRING +Usage: $this [-h] RELEASE_VERSION GPG_KEY Options: -h show this help and exit Example: - $this v0.1.2 "Jane Doe " ~/.gnupg/secring.gpg + $this v0.1.2 "Jane Doe " NOTE: The GPG key should be associated with the signer's Github account. - -NOTE: Helm is not compatible with GnuPG v2 and you need to export the secret - keys in order for Helm to be able to sign the package: - - gpg --export-secret-keys > ~/.gnupg/secring.gpg - EOF } +sign_helm_chart() { + local chart="$1" + echo "Signing Helm chart $chart" + local sha256=`openssl dgst -sha256 "$chart" | awk '{ print $2 }'` + local yaml=`tar xf $chart -O node-feature-discovery/Chart.yaml` + echo "$yaml +... +files: + $chart: sha256:$sha256" | gpg -u "$key" --clearsign -o "$chart.prov" +} + # # Parse command line # @@ -41,11 +46,11 @@ done shift "$((OPTIND - 1))" # Check that no extra args were provided -if [ $# -ne 3 ]; then - if [ $# -lt 3 ]; then +if [ $# -ne 2 ]; then + if [ $# -lt 2 ]; then echo -e "ERROR: too few arguments\n" else - echo -e "ERROR: unknown arguments: ${@:4}\n" + echo -e "ERROR: unknown arguments: ${@:3}\n" fi usage exit 1 @@ -53,8 +58,7 @@ fi release=$1 key="$2" -keyring="$3" -shift 3 +shift 2 container_image=k8s.gcr.io/nfd/node-feature-discovery:$release @@ -107,12 +111,11 @@ sed -e s'!"nfd\.repo",.*,!"nfd.repo", "k8s.gcr.io/nfd/node-feature-discovery",!' # # Create release assets to be uploaded # -helm package deployment/node-feature-discovery/ --version $semver --sign \ - --key "$key" --keyring "$keyring" +helm package deployment/node-feature-discovery/ --version $semver chart_name="node-feature-discovery-chart-$semver.tgz" mv node-feature-discovery-$semver.tgz $chart_name -mv node-feature-discovery-$semver.tgz.prov $chart_name.prov +sign_helm_chart $chart_name cat << EOF