File 0105-Add-github-release-sync.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 0105-Add-github-release-sync.patch of Package erlang

From 3113ca6094d0f2fb479d79e6a03410f5b312a920 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Thu, 24 Sep 2020 15:35:52 +0200
Subject: [PATCH] Add github release sync

This commit adds a github workflow that periodically syncs
the contents of erlang.org/download with
https://github.com/erlang/otp/releases.

This is done for any release > OTP-21.
---
 .github/workflows/sync-github-releases.yaml |  33 ++++
 scripts/sync-github-releases.sh             | 170 ++++++++++++++++++++
 2 files changed, 203 insertions(+)
 create mode 100644 .github/workflows/sync-github-releases.yaml
 create mode 100755 scripts/sync-github-releases.sh

diff --git a/.github/workflows/sync-github-releases.yaml b/.github/workflows/sync-github-releases.yaml
new file mode 100644
index 0000000000..2a88558d6b
--- /dev/null
+++ b/.github/workflows/sync-github-releases.yaml
@@ -0,0 +1,33 @@
+name: Sync all github releases with erlang.org
+
+## Update the base image every day
+on:
+  workflow_dispatch:
+  schedule:
+  ## In UTC
+  - cron: '*/30 * * * *'
+
+## Build base images to be used by other github workflows
+jobs:
+
+  # Wait for up to a minute for previous runs to complete, abort if not done by then
+  pre-ci:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    steps:
+      - name: 'Block Concurrent Executions'
+        uses: softprops/turnstyle@v1
+        with:
+          poll-interval-seconds: 10
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  sync-releases:
+    if: github.repository == 'erlang/otp'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Sync releases
+        run: >
+          scripts/sync-github-releases.sh ${{ github.repository }}
+          "Bearer ${{ secrets.GITHUB_TOKEN }}" "^2[123]\\..*" 25m
diff --git a/scripts/sync-github-releases.sh b/scripts/sync-github-releases.sh
new file mode 100755
index 0000000000..8b9cef6744
--- /dev/null
+++ b/scripts/sync-github-releases.sh
@@ -0,0 +1,170 @@
+#!/bin/bash
+
+## This bash script takes 4 arguments
+##  1: Repository on github to sync to
+##  2: The auth token to be used (either "token TheToken" or "Bearer TheToken")
+##  3: A regexp for which releases to sync
+##  4: Optional timeout for the rsync command when it should stop syncing
+##
+## This script downloads artifacts from erlang.org/download and then publishes
+## them to the release for the corresponding TAG on github. If there is no release
+## then a release is created with the README as the body of the release.
+##
+## The script does not keep release artifacts up to date, so if an artifact is changed
+## on erlang.org then it will not be automatically updated on github.
+##
+## The reason why this is a polling script and not triggered when a new tag is pushed
+## is because when the TAG is pushed to github it is not guarenteed that the
+## corresponding artifacts will be available on erlang.org.
+
+set -e
+
+REPOSITORY=${1}
+TOKEN=${2}
+RELEASE_FILTER=${3}
+TIME_LIMIT=${4:-120m}
+HDR=(-H "Authorization: ${TOKEN}")
+REPO="https://api.github.com/repos/${REPOSITORY}"
+
+_json_escape () {
+    printf '```\n%s\n```' "${1}" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))'
+}
+
+_strip_name() {
+    echo ${1} | sed -e 's/^OTP[-_]//g'
+}
+
+_curl_get() {
+    curl --silent "${HDR[@]}" "${@}"
+}
+
+_curl_post() {
+    curl -o /dev/null --silent --fail --show-error -X POST "${HDR[@]}" \
+         -H "Accept: application/vnd.github.v3+json" "${@}"
+}
+
+RI=""
+ALL_TAGS=""
+CREATE_RELEASE=""
+TAG_URL="${REPO}/tags?per_page=100"
+
+## This function is used to loop over the pagianated results from github tags
+## It sets TAGS to be the the json from the current page of tags
+_next_page() {
+    TAGS=`curl -s -H "${HDR[@]}" ${TAG_URL}`
+    ## In the "Link:" header from github we get the link for the next page.
+    ## An example link header:
+    ##   link: <https://api.github.com/repositories/843890/tags?per_page=100&page=2>; rel="next", <https://api.github.com/repositories/843890/tags?per_page=100&page=4>; rel="last"
+    TAG_URL=`curl -s -I -H "${HDR[@]}" ${TAG_URL} | grep "^link:" | sed -n 's/link:.* <$[^>]\+$>; rel="next".*/\1/p'`
+}
+
+## First we fetch all tags and releases and build a list of all resources
+## that we should rsync from erlang.org. We only want to do one call to
+## rsync for all files as otherwise erlang.org will rate-limit us.
+while [ "${TAG_URL}" != "" ]; do
+    _next_page
+
+    ## Loop over all tags, we base64 encode each element in the array
+    ## in order to make the bash for loop work
+    for row in $(echo "${TAGS}" | jq -r '.[] | @base64'); do
+        _row() {
+            echo ${row} | base64 --decode | jq -r ${1}
+        }
+        name=$(_row '.name')
+        stripped_name=$(_strip_name ${name})
+
+        if echo ${stripped_name} | grep -E "${RELEASE_FILTER}" > /dev/null; then
+            ALL_TAGS="${ALL_TAGS} ${name}"
+            RELEASE=$(_curl_get "${REPO}/releases/tags/${name}")
+            if ! echo "${RELEASE}" | jq -er ".name" > /dev/null; then
+                CREATE_RELEASE="${CREATE_RELEASE} ${name}"
+                RI="*${stripped_name}* ${RI}"
+                echo "Create release ${name}"
+            else
+                _asset() {
+                    local filename=${1}
+                    local remotename=${2:-${filename}}
+                    if ! echo "${RELEASE}" | jq -er ".assets[] | select(.name == \"${filename}\")" > /dev/null; then
+                        echo "Sync ${remotename} for ${name}"
+                        RI="${remotename} ${RI}"
+                    fi
+                }
+                _asset "${name}.README" "${name}.README otp_src_${stripped_name}.readme"
+                _asset "otp_src_${stripped_name}.tar.gz" ""
+                _asset "otp_doc_html_${stripped_name}.tar.gz" ""
+                _asset "otp_doc_man_${stripped_name}.tar.gz" ""
+                _asset "otp_win32_${stripped_name}.exe" ""
+                _asset "otp_win64_${stripped_name}.exe" ""
+            fi
+        fi
+    done
+done
+
+RINCLUDE=""
+for i in ${RI}; do
+    RINCLUDE="--include=${i} ${RINCLUDE}"
+done
+
+## rsync the proper files, we will use which files have been
+## synced to determine which artifacts we should upload.
+## There is a timelimit here so that github actions will not
+## timeout
+! timeout ${TIME_LIMIT} rsync --archive --verbose --compress ${RINCLUDE} --exclude='*' \
+     erlang.org::erlang-download downloads
+
+## Rename all .readme files to .README
+for name in ${ALL_TAGS}; do
+    stripped_name=$(_strip_name ${name})
+    if [ -s "downloads/otp_src_${stripped_name}.readme" ]; then
+        mv downloads/otp_src_${stripped_name}.readme downloads/${name}.README
+    fi
+done
+
+## All tags that do not have a release we create a release for
+## using the readme as the body text if a readme is available.
+for name in ${CREATE_RELEASE}; do
+    echo "Create release for ${name}"
+    stripped_name=$(_strip_name ${name})
+    if [ -s "downloads/${name}.README" ]; then
+        README=`cat downloads/${name}.README`
+    else
+        README=""
+    fi
+    if echo "${README}" | grep "HIGHLIGHTS" > /dev/null; then
+        ## We have highlights, so only use those as the body
+
+        ## This awk script is a hack.
+        ## It counts the number of lines that start with '---' and
+        ## then outputs any text after the first '---' until the 7th.
+        README=`echo "${README}" | awk 'BEGIN{ echo=0 } { if ( $1 ~ /^---/ ) { echo++ } if ( echo > 0 && echo < 7 ) { print $0 } }'`
+    fi
+    if [ "${README}" != "" ]; then
+        RM=$(_json_escape "${README}")
+        BODY=", \"body\":${RM}"
+    else
+        BODY=""
+    fi
+    $(_curl_post "${REPO}/releases" -d '{"tag_name":"'"${name}"'", "name":"OTP '"${stripped_name}\"${BODY}}")
+done
+
+## Upload all assets for tags
+for name in ${ALL_TAGS}; do
+    echo "Upload artifacts for ${name}"
+    stripped_name=$(_strip_name ${name})
+    RELEASE=$(_curl_get "${REPO}/releases/tags/${name}")
+    UPLOAD_URL=`echo "${RELEASE}" | jq -r ".upload_url" | sed 's/{.*//'`
+    _upload() {
+        if [ -s downloads/${1} ]; then
+            echo "Upload ${1}"
+            $(_curl_post -H "Content-Type: ${2}" \
+                         "${UPLOAD_URL}?name=${1}" \
+                         --data-binary "@downloads/${1}")
+        fi
+    }
+    _upload "${name}.README" "text"
+    _upload "otp_src_${stripped_name}.tar.gz" "application/gzip"
+    _upload "otp_doc_html_${stripped_name}.tar.gz" "application/gzip"
+    _upload "otp_doc_man_${stripped_name}.tar.gz" "application/gzip"
+    _upload "otp_win32_${stripped_name}.exe" "application/x-msdownload"
+    _upload "otp_win64_${stripped_name}.exe" "application/x-msdownload"
+done
-- 
2.26.2

Places

File 0105-Add-github-release-sync.patch of Package erlang

Places