[TESTS] upgrade tests for storage

(cherry picked from commit 884ca63738) (cherry picked from commit 0a45d9c37b) [TESTS] upgrade tests for storage (squash) relative paths (cherry picked from commit 3bb19285f3) (cherry picked from commit c640c09e61) (cherry picked from commit 40ffe2d226) [UPGRADE] S3 storage and fixtures (cherry picked from commit c466c9c657) (cherry picked from commit e80abbe2cd) (cherry picked from commit 50a47df1d1) [UPGRADE] add sanity checks for [storage*] (squash) speedup upgrade tests (cherry picked from commit f578279cfe) (cherry picked from commit 6dd677f151) (cherry picked from commit ee7e7eca27) (cherry picked from commit 879ffca697) (cherry picked from commit 60b33cc5af) (cherry picked from commit 3ba984b242) (cherry picked from commit fceff4ca7b)
2023-08-11 19:12:44 +02:00 · 2023-08-11 19:12:44 +02:00 · e0d732a4e8
parent 52cc6c26fc
commit e0d732a4e8
12 changed files with 1238 additions and 0 deletions
--- a/.forgejo/upgrades/default-app.ini
+++ b/.forgejo/upgrades/default-app.ini
@ -0,0 +1,30 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+PATH = ${WORK_PATH}/forgejo.db
+
+[log]
+MODE = file
+LEVEL = trace
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
--- a/.forgejo/upgrades/fixtures.sh
+++ b/.forgejo/upgrades/fixtures.sh
@ -0,0 +1,187 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+#ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+#
+# one pixel scaled to 290x290 because that's what versions lower or equal to v1.19.4-0 want
+# by default and any other size will be transformed which make it difficult to compare.
+#
+ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg=="
+
+function fixture_get_paths_s3() {
+    local path=$1
+
+    (
+        echo -n $path/
+        mc ls --quiet --recursive testS3/$path | sed -e 's/.* //'
+    ) > $DIR/path
+}
+
+function fixture_get_paths_local() {
+    local path=$1
+    local work_path=$DIR/forgejo-work-path
+
+    ( cd $work_path ; find $path -type f) > $DIR/path
+}
+
+function fixture_get_one_path() {
+    local storage=$1
+    local path=$2
+
+    fixture_get_paths_$storage $path
+
+    if test $(wc -l < $DIR/path) != 1 ; then
+        echo expected one path but got
+        cat $DIR/path
+        return 1
+    fi
+    cat $DIR/path
+}
+
+function fixture_repo_archive_create() {
+    retry curl -f -sS http://${HOST_PORT}/root/fixture/archive/main.zip -o /dev/null
+}
+
+function fixture_repo_archive_assert_s3() {
+    mc ls --recursive testS3/forgejo/repo-archive | grep --quiet '.zip$'
+}
+
+function fixture_repo_archive_assert_local() {
+    local path=$1
+    local work_path=$DIR/forgejo-work-path
+
+    find $work_path/$path | grep --quiet '.zip$'
+}
+
+function fixture_lfs_create() {
+    (
+        cd $DIR/fixture
+        git lfs track "*.txt"
+        echo CONTENT > file.txt
+        git add .
+        git commit -m 'lfs files'
+        git push
+    )
+}
+
+function fixture_lfs_assert_s3() {
+    local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7)
+    test "$content" = CONTENT
+}
+
+function fixture_lfs_assert_local() {
+    local path=$1
+    local work_path=$DIR/forgejo-work-path
+
+    local content=$(mc cat $work_path/$path/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7)
+    test "$content" = CONTENT
+}
+
+function fixture_packages_create() {
+    echo PACKAGE_CONTENT > $DIR/fixture/package
+    $work_path/forgejo-api -X DELETE http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt || true
+    $work_path/forgejo-api --upload-file $DIR/fixture/package http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt
+}
+
+function fixture_packages_assert_s3() {
+    local content=$(mc cat testS3/forgejo/packages/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17)
+    test "$content" = PACKAGE_CONTENT
+}
+
+function fixture_packages_assert_local() {
+    local path=$1
+
+    local content=$(cat $work_path/$path/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17)
+    test "$content" = PACKAGE_CONTENT
+}
+
+function fixture_avatars_create() {
+    echo -n $ONEPIXEL | base64 --decode > $DIR/avatar.png
+    $work_path/forgejo-client --form avatar=@$DIR/avatar.png http://${HOST_PORT}/user/settings/avatar
+}
+
+function fixture_avatars_assert_s3() {
+    local filename=$(fixture_get_one_path s3 forgejo/avatars)
+    local content=$(mc cat testS3/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_avatars_assert_local() {
+    local path=$1
+
+    local filename=$(fixture_get_one_path local $path)
+    local content=$(cat $work_path/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_repo_avatars_create() {
+    echo -n $ONEPIXEL | base64 --decode > $DIR/repo-avatar.png
+    $work_path/forgejo-client --form avatar=@$DIR/repo-avatar.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/settings/avatar
+    # v1.21 only
+    #$work_path/forgejo-api -X POST --data-raw '{"body":"'$avatar'"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/avatar
+}
+
+function fixture_repo_avatars_assert_s3() {
+    local filename=$(fixture_get_one_path s3 forgejo/repo-avatars)
+    local content=$(mc cat testS3/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_repo_avatars_assert_local() {
+    local path=$1
+
+    local filename=$(fixture_get_one_path local $path)
+    local content=$(cat $work_path/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_attachments_create_1_18() {
+    echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png
+    $work_path/forgejo-client --trace-ascii - --form file=@$DIR/attachment.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/issues/attachments
+}
+
+function fixture_attachments_create() {
+    if $work_path/forgejo-api http://${HOST_PORT}/api/v1/version | grep --quiet --fixed-strings 1.18. ; then
+        fixture_attachments_create_1_18
+        return
+    fi
+    id=$($work_path/forgejo-api --data-raw '{"title":"TITLE"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues | jq .id)
+    echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png
+    $work_path/forgejo-client -H @$DIR/forgejo-work-path/forgejo-header --form name=attachment.png --form attachment=@$DIR/attachment.png http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues/$id/assets
+}
+
+function fixture_attachments_assert_s3() {
+    local filename=$(fixture_get_one_path s3 forgejo/attachments)
+    local content=$(mc cat testS3/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_attachments_assert_local() {
+    local path=$1
+
+    local filename=$(fixture_get_one_path local $path)
+    local content=$(cat $work_path/$filename | base64 -w0)
+    test "$content" = "$ONEPIXEL"
+}
+
+function fixture_create() {
+    local work_path=$DIR/forgejo-work-path
+
+    rm -fr $DIR/fixture
+    mkdir -p $DIR/fixture
+    (
+        cd $DIR/fixture
+        git init
+        git checkout -b main
+        git remote add origin http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}
+        git config user.email root@example.com
+        git config user.name username
+        echo SOMETHING > README
+        git add README
+        git commit -m 'initial commit'
+        git push --set-upstream --force origin main
+    )
+    for fun in ${STORAGE_FUN} ; do
+        fixture_${fun}_create
+    done
+}
--- a/.forgejo/upgrades/legagy-relative-app.ini
+++ b/.forgejo/upgrades/legagy-relative-app.ini
@ -0,0 +1,32 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+LFS_CONTENT_PATH = relative-lfs
+
+[database]
+DB_TYPE = sqlite3
+PATH = ${WORK_PATH}/forgejo.db
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[picture]
+AVATAR_UPLOAD_PATH = relative-avatars
+REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars
--- a/.forgejo/upgrades/merged-app.ini
+++ b/.forgejo/upgrades/merged-app.ini
@ -0,0 +1,32 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
+
+[storage]
+PATH = ${WORK_PATH}/merged
--- a/.forgejo/upgrades/misplace-app.ini
+++ b/.forgejo/upgrades/misplace-app.ini
@ -0,0 +1,59 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/elsewhere
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
+
+[attachment]
+
+[storage.attachments]
+PATH = ${WORK_PATH}/data/attachments
+
+[lfs]
+
+[storage.lfs]
+PATH = ${WORK_PATH}/data/lfs
+
+[avatar]
+
+[storage.avatars]
+PATH = ${WORK_PATH}/data/avatars
+
+[repo-avatar]
+
+[storage.repo-avatars]
+PATH = ${WORK_PATH}/data/repo-avatars
+
+[repo-archive]
+
+[storage.repo-archive]
+PATH = ${WORK_PATH}/data/repo-archive
+
+[packages]
+
+[storage.packages]
+PATH = ${WORK_PATH}/data/packages
--- a/.forgejo/upgrades/misplace-s3-app.ini
+++ b/.forgejo/upgrades/misplace-s3-app.ini
@ -0,0 +1,89 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/elsewhere
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
+
+[attachment]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
+
+[lfs]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
+
+[repo-avatar]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
+
+[avatar]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
+
+[repo-archive]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
+
+[packages]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
--- a/.forgejo/upgrades/relative-app.ini
+++ b/.forgejo/upgrades/relative-app.ini
@ -0,0 +1,44 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[attachment]
+PATH = relative-attachments
+
+[lfs]
+PATH = relative-lfs
+
+[avatar]
+PATH = relative-avatars
+
+[repo-avatar]
+PATH = relative-repo-avatars
+
+[repo-archive]
+PATH = relative-repo-archive
+
+[packages]
+PATH = relative-packages
--- a/.forgejo/upgrades/specific-app.ini
+++ b/.forgejo/upgrades/specific-app.ini
@ -0,0 +1,47 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/elsewhere
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
+
+[attachment]
+PATH = ${WORK_PATH}/data/attachments
+
+[lfs]
+PATH = ${WORK_PATH}/data/lfs
+
+[avatar]
+PATH = ${WORK_PATH}/data/avatars
+
+[repo-avatar]
+PATH = ${WORK_PATH}/data/repo-avatars
+
+[repo-archive]
+PATH = ${WORK_PATH}/data/repo-archive
+
+[packages]
+PATH = ${WORK_PATH}/data/packages
--- a/.forgejo/upgrades/stable-s3-app.ini
+++ b/.forgejo/upgrades/stable-s3-app.ini
@ -0,0 +1,39 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/elsewhere
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[actions]
+ENABLED = true
+
+[storage]
+STORAGE_TYPE = minio
+SERVE_DIRECT = false
+MINIO_ENDPOINT = 127.0.0.1:9000
+MINIO_ACCESS_KEY_ID = 123456
+MINIO_SECRET_ACCESS_KEY = 12345678
+MINIO_BUCKET = forgejo
+MINIO_LOCATION = us-east-1
+MINIO_USE_SSL = false
--- a/.forgejo/upgrades/storage-relative-app.ini
+++ b/.forgejo/upgrades/storage-relative-app.ini
@ -0,0 +1,44 @@
+RUN_MODE = prod
+WORK_PATH = ${WORK_PATH}
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+HTTP_PORT = 3000
+SSH_LISTEN_PORT = 2222
+LFS_START_SERVER = true
+
+[database]
+DB_TYPE = sqlite3
+
+[log]
+MODE = file
+LEVEL = debug
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[storage.attachments]
+PATH = relative-attachments
+
+[storage.lfs]
+PATH = relative-lfs
+
+[storage.avatars]
+PATH = relative-avatars
+
+[storage.repo-avatars]
+PATH = relative-repo-avatars
+
+[storage.repo-archive]
+PATH = relative-repo-archive
+
+[storage.packages]
+PATH = relative-packages
--- a/.forgejo/upgrades/test-upgrade.sh
+++ b/.forgejo/upgrades/test-upgrade.sh
@ -0,0 +1,590 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+#
+# Debug loop from the source tree:
+#
+# ./.forgejo/upgrades/test-upgrade.sh dependencies
+# ./.forgejo/upgrades/test-upgrade.sh build_all
+# VERBOSE=true ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails
+#
+# Everything happens in /tmp/forgejo-upgrades
+#
+
+PREFIX===============
+HOST_PORT=0.0.0.0:3000
+STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars"
+STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars"
+DIR=/tmp/forgejo-upgrades
+if ${VERBOSE:-false} ; then
+    set -ex
+    PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
+else
+    set -e
+fi
+SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+: ${FORGEJO_USER:=root}
+: ${FORGEJO_REPO:=fixture}
+: ${FORGEJO_PASSWORD:=admin1234}
+
+source $SELF_DIR/fixtures.sh
+
+function maybe_sudo() {
+    if test $(id -u) != 0 ; then
+        SUDO=sudo
+    fi
+}
+
+function log_info() {
+    echo "$PREFIX $@"
+}
+
+function dependencies() {
+    maybe_sudo
+    if ! which curl daemon jq git-lfs > /dev/null ; then
+        $SUDO apt-get install -y -qq curl daemon git-lfs jq
+    fi
+
+    if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio  > /dev/null ; then
+        $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
+        $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-08-23T10-07-06Z -o /usr/local/bin/minio
+    fi
+    if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio  > /dev/null ; then
+        $SUDO chmod +x /usr/local/bin/mc
+        $SUDO chmod +x /usr/local/bin/minio
+    fi
+
+    if ! test -f /usr/local/bin/garage > /dev/null ; then
+        $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage
+    fi
+    if ! test -x /usr/local/bin/garage  > /dev/null ; then
+        $SUDO chmod +x /usr/local/bin/garage
+    fi
+}
+
+function build() {
+    local version=$1
+    local semver=$2
+
+    if ! test -f $DIR/forgejo-$version ; then
+        mkdir -p $DIR
+        make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea
+        mv gitea $DIR/forgejo-$version
+    fi
+}
+
+function build_all() {
+    test -f Makefile
+    build 1.21.0-0 6.0.0+0-gitea-1.21.0
+}
+
+function retry() {
+    rm -f $DIR/wait-for.out
+    success=false
+    for delay in 1 1 5 5 15 ; do
+        if "$@" >> $DIR/wait-for.out 2>&1 ; then
+            success=true
+            break
+        fi
+        cat $DIR/wait-for.out
+        echo waiting $delay
+        sleep $delay
+    done
+    if test $success = false ; then
+        cat $DIR/wait-for.out
+        return 1
+    fi
+}
+
+function download() {
+    local version=$1
+
+    if ! test -f $DIR/forgejo-$version ; then
+        mkdir -p $DIR
+        wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64
+        chmod +x $DIR/forgejo-$version
+    fi
+}
+
+function cleanup_logs() {
+    local work_path=$DIR/forgejo-work-path
+
+    rm -f $DIR/*.log
+    rm -f $work_path/log/*.log
+}
+
+function clobber() {
+    rm -fr /tmp/forgejo-upgrades
+}
+
+function start_forgejo() {
+    local version=$1
+
+    download $version
+    local work_path=$DIR/forgejo-work-path
+    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path
+    if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then
+        cat $DIR/*.log
+        cat $work_path/log/*.log
+        return 1
+    fi
+    create_user $version
+}
+
+function start_minio() {
+    mkdir -p $DIR/minio
+    daemon --chdir=$DIR --unsafe \
+           --env="PATH=$PATH" \
+           --env=MINIO_ROOT_USER=123456 \
+           --env=MINIO_ROOT_PASSWORD=12345678 \
+           --env=MINIO_VOLUMES=$DIR/minio \
+           --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server
+    retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678
+}
+
+function start_garage() {
+    mkdir -p $DIR/garage/{data,meta}
+    cat > $DIR/garage/garage.toml <<EOF
+metadata_dir = "$DIR/garage/meta"
+data_dir = "$DIR/garage/data"
+db_engine = "lmdb"
+
+replication_mode = "none"
+
+rpc_bind_addr = "127.0.0.1:3901"
+rpc_public_addr = "127.0.0.1:3901"
+rpc_secret = "$(openssl rand -hex 32)"
+
+[s3_api]
+s3_region = "us-east-1"
+api_bind_addr = "127.0.0.1:9000"
+root_domain = ".s3.garage.localhost"
+
+[s3_web]
+bind_addr = "127.0.0.1:3902"
+root_domain = ".web.garage.localhost"
+index = "index.html"
+
+[k2v_api]
+api_bind_addr = "127.0.0.1:3904"
+
+[admin]
+api_bind_addr = "127.0.0.1:3903"
+admin_token = "$(openssl rand -base64 32)"
+EOF
+
+    daemon --chdir=$DIR --unsafe \
+           --env="PATH=$PATH" \
+           --env=RUST_LOG=garage_api=debug \
+           --pidfile=$DIR/garage-pid --errlog=$DIR/garage-err.log --output=$DIR/garage-out.log -- /usr/local/bin/garage -c $DIR/garage/garage.toml server
+
+    retry garage -c $DIR/garage/garage.toml status
+    garage -c $DIR/garage/garage.toml layout assign -z dc1 -c 1 $(garage -c $DIR/garage/garage.toml status | tail -1 | grep -o '[0-9a-z]*' | head -1)
+    ver=$(garage -c $DIR/garage/garage.toml layout show | grep -oP '(?<=Current cluster layout version: )\d+')
+    garage -c $DIR/garage/garage.toml layout apply --version $((ver+1))
+    garage -c $DIR/garage/garage.toml key info test || garage -c $DIR/garage/garage.toml key import -n test 123456 12345678
+    garage -c $DIR/garage/garage.toml key allow --create-bucket test
+    retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678
+}
+
+function start_s3() {
+    local s3_backend=$1
+
+    start_$s3_backend
+}
+
+function start() {
+    local version=$1
+    local s3_backend=${2:-minio}
+
+    start_s3 $s3_backend
+    start_forgejo $version
+}
+
+function create_user() {
+    local version=$1
+
+    local work_path=$DIR/forgejo-work-path
+
+    if test -f $work_path/forgejo-token; then
+        return
+    fi
+
+    local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path"
+    $cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email "$FORGEJO_USER@example.com"
+    local scopes="--scopes all"
+    if echo $version | grep --quiet 1.18. ; then
+        scopes=""
+    fi
+
+    #
+    # forgejo-cli is to use with api/v1 enpoints
+    #
+    # tail -1 is because there are logs creating noise in the output in v1.19.4-0
+    #
+    $cli admin user generate-access-token -u $FORGEJO_USER --raw $scopes | tail -1 > $work_path/forgejo-token
+    ( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header
+    ( echo "#!/bin/sh" ; echo 'curl -f -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api
+    $work_path/forgejo-api http://${HOST_PORT}/api/v1/version
+
+    #
+    # forgejo-client is to use with web endpoints
+    #
+    #
+    # login and obtain a CSRF, all stored in the cookie file
+    #
+    ( echo "#!/bin/sh" ; echo 'curl --cookie-jar '$DIR/cookies' --cookie '$DIR/cookies' -f -sS "$@"' ) > $work_path/forgejo-client-update-cookies && chmod +x $work_path/forgejo-client-update-cookies
+    $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null
+    $work_path/forgejo-client-update-cookies --verbose -X POST --data user_name=${FORGEJO_USER} --data password=${FORGEJO_PASSWORD} http://${HOST_PORT}/user/login >& $DIR/login.html
+    $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null
+    local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DIR/cookies)
+    #
+    # use the cookie file but do not modify it
+    #
+    ( echo "#!/bin/sh" ; echo 'curl --cookie '$DIR/cookies' -H "X-Csrf-Token: '$csrf'" -f -sS "$@"' ) > $work_path/forgejo-client && chmod +x $work_path/forgejo-client
+}
+
+function stop_daemon() {
+    local daemon=$1
+
+    if test -f $DIR/$daemon-pid ; then
+        local pid=$(cat $DIR/$daemon-pid)
+        kill -TERM $pid
+        pidwait $pid || true
+        for delay in 1 1 2 2 5 5 ; do
+            if ! test -f $DIR/$daemon-pid ; then
+                break
+            fi
+            sleep $delay
+        done
+        ! test -f $DIR/$daemon-pid
+    fi
+}
+
+function stop() {
+    stop_daemon forgejo
+    stop_daemon minio
+    stop_daemon garage
+
+    cleanup_logs
+}
+
+function reset_forgejo() {
+    local config=$1
+    local work_path=$DIR/forgejo-work-path
+    rm -fr $work_path
+    mkdir -p $work_path
+    WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini
+}
+
+function reset_minio() {
+    rm -fr $DIR/minio
+}
+
+function reset_garage() {
+    rm -fr $DIR/garage
+}
+
+function reset() {
+    local config=$1
+    reset_forgejo $config
+    reset_minio
+    reset_garage
+}
+
+function verify_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        test -d $work_path/data/$path
+    done
+}
+
+function cleanup_storage() {
+    local work_path=$DIR/forgejo-work-path
+
+    for path in ${STORAGE_PATHS} ; do
+        rm -fr $work_path/data/$path
+    done
+}
+
+function test_downgrade_1.20.2_fails() {
+    local work_path=$DIR/forgejo-work-path
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
+
+    log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails"
+    stop
+    reset default
+    start 1.20.3-0
+    stop
+    download 1.20.2-0
+    timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true
+    if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then
+        cat $work_path/log/forgejo.log
+        return 1
+    fi
+}
+
+function test_bug_storage_merged() {
+    local work_path=$DIR/forgejo-work-path
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
+
+    log_info "using < 1.20.3-0 and [storage].PATH merge all storage"
+    for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
+        stop
+        reset merged
+        start $version
+        for path in ${STORAGE_PATHS} ; do
+            ! test -d $work_path/data/$path
+        done
+        for path in ${STORAGE_PATHS} ; do
+            ! test -d $work_path/merged/$path
+        done
+        test -d $work_path/merged
+    done
+    stop
+
+    log_info "upgrading from 1.20.2-0 with [storage].PATH fails"
+    download 1.20.3-0
+    timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
+    if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then
+        cat $work_path/log/forgejo.log
+        return 1
+    fi
+}
+
+function test_bug_storage_relative_path() {
+    local work_path=$DIR/forgejo-work-path
+
+    log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH"
+    for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
+        stop
+        reset legagy-relative
+        start $version
+        test -d $work_path/relative-lfs
+        test -d $work_path/relative-avatars
+        test -d $work_path/relative-repo-avatars
+    done
+
+    log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH"
+    for version in 1.20.3-0 1.21.0-0 ; do
+        stop
+        reset legagy-relative
+        start $version
+        test -d $work_path/data/relative-lfs
+        test -d $work_path/data/relative-avatars
+        test -d $work_path/data/relative-repo-avatars
+    done
+
+    log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH"
+    for version in 1.20.3-0 1.21.0-0 ; do
+        stop
+        reset storage-relative
+        start $version
+        for path in ${STORAGE_PATHS} ; do
+            test -d $work_path/data/relative-$path
+        done
+    done
+
+    log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent"
+    for version in 1.20.2-0 ; do
+        stop
+        reset storage-relative
+        start $version
+        test -d $work_path/data/packages
+        test -d $work_path/relative-repo-archive
+        test -d $work_path/relative-attachments
+        test -d $work_path/relative-lfs
+        test -d $work_path/data/avatars
+        test -d $work_path/data/repo-avatars
+    done
+
+    log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent"
+    for version in 1.18.5-0 1.19.4-0 ; do
+        stop
+        reset storage-relative
+        start $version
+        test -d $work_path/relative-packages
+        test -d $work_path/relative-repo-archive
+        test -d $work_path/relative-attachments
+        test -d $work_path/data/lfs
+        test -d $work_path/data/avatars
+        test -d $work_path/data/repo-avatars
+    done
+
+    log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH"
+    for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do
+        stop
+        reset relative
+        start $version
+        for path in ${STORAGE_PATHS} ; do
+            test -d $work_path/relative-$path
+        done
+    done
+
+    log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH"
+    for version in 1.20.3-0 1.21.0-0 ; do
+        stop
+        reset relative
+        start $version
+        for path in ${STORAGE_PATHS} ; do
+            test -d $work_path/data/relative-$path
+        done
+    done
+
+    stop
+}
+
+function test_bug_storage_s3_misplace() {
+    local work_path=$DIR/forgejo-work-path
+    local s3_backend=${2:-minio}
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
+
+    for version in 1.20.2-0 1.20.3-0 ; do
+        log_info "Forgejo $version & $s3_backend"
+        stop
+        reset misplace-s3
+        start $version $s3_backend
+        fixture_create
+        for fun in ${STORAGE_FUN} ; do
+            fixture_${fun}_assert_s3
+        done
+    done
+
+    for version in 1.18.5-0 1.19.4-0 ; do
+        log_info "Forgejo $version & $s3_backend"
+        stop
+        reset misplace-s3
+        start $version $s3_backend
+        fixture_create
+        #
+        # some storage are in S3
+        #
+        fixture_attachments_assert_s3
+        fixture_lfs_assert_s3
+        #
+        # others are in local
+        #
+        fixture_repo_archive_assert_local elsewhere/repo-archive
+        fixture_avatars_assert_local elsewhere/avatars
+        fixture_packages_assert_local elsewhere/packages
+        fixture_repo_avatars_assert_local elsewhere/repo-avatars
+    done
+}
+
+function test_storage_stable_s3() {
+    local work_path=$DIR/forgejo-work-path
+    local s3_backend=${1:-minio}
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338"
+
+    for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 ; do
+        log_info "Forgejo $version & $s3_backend"
+        stop
+        reset stable-s3
+        start $version $s3_backend
+        fixture_create
+        for fun in ${STORAGE_FUN} ; do
+            fixture_${fun}_assert_s3
+        done
+    done
+}
+
+function test_bug_storage_misplace() {
+    local work_path=$DIR/forgejo-work-path
+
+    log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225"
+
+    log_info "using < 1.20 and conflicting sections misplace storage"
+    for version in 1.18.5-0 1.19.4-0 ; do
+        stop
+        reset misplace
+        start $version
+        #
+        # some storage are where they should be
+        #
+        test -d $work_path/data/packages
+        test -d $work_path/data/repo-archive
+        test -d $work_path/data/attachments
+        #
+        # others are under APP_DATA_PATH
+        #
+        test -d $work_path/elsewhere/lfs
+        test -d $work_path/elsewhere/avatars
+        test -d $work_path/elsewhere/repo-avatars
+    done
+
+    log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]"
+    for version in 1.20.2-0 ; do
+        stop
+        reset misplace
+        start $version
+        for path in ${STORAGE_PATHS} ; do
+            test -d $work_path/elsewhere/$path
+        done
+    done
+
+    stop
+
+    log_info "upgrading from 1.20.2-0 with conflicting sections fails"
+    download 1.20.3-0
+    timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
+    for path in ${STORAGE_PATHS} ; do
+        if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then
+            cat $work_path/log/forgejo.log
+            return 1
+        fi
+    done
+}
+
+function test_successful_upgrades() {
+    for config in default specific ; do
+        log_info "using $config app.ini"
+        reset $config
+
+        for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do
+            log_info "run $version"
+            cleanup_storage
+            start $version
+            verify_storage
+            stop
+        done
+    done
+}
+
+function run() {
+    local fun=$1
+    shift
+
+    echo Start running $fun
+    mkdir -p $DIR
+    > $DIR/$fun.out
+    tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" &
+    pid=$!
+    if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then
+        kill $pid
+        cat $DIR/$fun.out
+        echo Failure running $fun
+        return 1
+    fi
+    kill $pid
+    echo Success running $fun
+}
+
+function test_upgrades() {
+    run stop
+    run dependencies
+    run build_all
+    run test_successful_upgrades
+    run test_bug_storage_misplace
+    run test_bug_storage_merged
+    run test_downgrade_1.20.2_fails
+    run test_bug_storage_s3_misplace
+    run test_storage_stable_s3 minio
+    run test_storage_stable_s3 garage
+}
+
+"$@"
--- a/.forgejo/workflows/upgrade.yml
+++ b/.forgejo/workflows/upgrade.yml
@ -0,0 +1,45 @@
+name: upgrade
+
+on:
+  pull_request_review:
+  push:
+    branches:
+      - 'forgejo*'
+      - 'v*/forgejo*'
+
+jobs:
+  upgrade:
+    runs-on: docker
+    container:
+      image: codeberg.org/forgejo/test_env:main
+    steps:
+      - run: apt-get install -y -qq zstd
+
+      - name: cache S3 binaries
+        id: S3
+        uses: https://code.forgejo.org/actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/minio
+            /usr/local/bin/mc
+            /usr/local/bin/garage
+          key: S3
+
+      - name: skip if S3 cache hit
+        if: steps.S3.outputs.cache-hit != 'true'
+        run: echo no hit
+
+      - uses: https://code.forgejo.org/actions/checkout@v3
+      - uses: https://code.forgejo.org/actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - run: |
+          git config --add safe.directory '*'
+          chown -R gitea:gitea . /go
+      - run: |
+          su gitea -c 'make deps-backend'
+      - run: |
+          script=$(pwd)/.forgejo/upgrades/test-upgrade.sh
+          $script run dependencies
+          $script clobber
+          su gitea -c "$script test_upgrades"