containerized Jenkins (#179)

Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: https://dev.hostsharing.net/hostsharing/hs.hsadmin.ng/pulls/179
Reviewed-by: Timotheus Pokorra <timotheus.pokorra@hostsharing.net>

.aliases

@@ -129,8 +129,7 @@ function _gwTest() {
     # delierately in separate gradlew-calls to avoid Testcontains-PostgreSQL problem spillover
     time (_gwTest1 unitTest "$@" &&
         _gwTest1 officeIntegrationTest bookingIntegrationTest hostingIntegrationTest "$@" &&
-        _gwTest1 scenarioTest "$@" &&
+        _gwTest1 scenarioTest "$@" && _gwTest1 migrationTest "$@");
-        _gwTest1 importHostingAssets "$@");
   elif [ $# -eq 0 ] || [[ $1 == -* ]]; then
     time _gwTest1 test "$@";
   else

Jenkins/Dockerfile

@@ -0,0 +1,15 @@
+FROM jenkins/jenkins:lts-jdk21
+
+USER root
+
+# Docker CLI installieren
+RUN apt-get update && apt-get install -y docker.io && usermod -aG docker jenkins
+
+# grant user jenkins access to /var/run/docker.sock
+RUN usermod -aG messagebus jenkins
+
+# install plugins
+COPY Jenkins.plugins /usr/share/jenkins/ref/plugins.txt
+RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt
+
+USER jenkins

Jenkins/Jenkins.plugins

@@ -0,0 +1,8 @@
+git
+workflow-aggregator
+pipeline-github-lib
+docker-workflow
+credentials
+git-client
+blueocean
+coverage

Jenkins/Jenkinsfile

@@ -0,0 +1,138 @@
+pipeline {
+    parameters {
+        string(name: 'AGENT_CPUS', defaultValue: '2.5', description: 'CPU limit for the build agent')
+        string(name: 'AGENT_NETWORK', defaultValue: 'host', description: 'Network to be used for build agent')
+        booleanParam(name: 'QUICK_RUN', defaultValue: false, description: 'false: all stages but slow, true: just some stages and fast')
+    }
+    agent {
+        dockerfile {
+            filename 'Jenkins/agent/Dockerfile'
+            args """--user root --network ${params.AGENT_NETWORK}
+                    --volume /var/run/docker.sock:/var/run/docker.sock
+                    --memory=8g --cpus=${params.AGENT_CPUS}"""
+       }
+    }
+
+    environment {
+        GRADLE_USER_HOME = "${env.WORKSPACE}/.gradle-cache"
+        DOCKER_HOST = 'unix:///var/run/docker.sock'
+        HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin'
+        HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted'
+        HSADMINNG_MIGRATION_DATA_PATH = 'migration'
+        TESTCONTAINERS_RYUK_DISABLED = true
+        TESTCONTAINERS_LOG_LEVEL = "DEBUG"
+    }
+
+    triggers {
+        pollSCM('H/1 * * * *')
+    }
+
+    stages {
+        stage('Detect Docker Environment') {
+            steps {
+                sh '''#!/bin/bash +x
+                    if command -v docker >/dev/null 2>&1; then
+                      if docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'rootless'; then
+                        echo "🟡 Docker daemon is running in ROOTLESS mode"
+                      else
+                        echo "🟢 Docker daemon is running in ROOTFUL mode"
+                      fi
+                    else
+                      echo "❌ Docker CLI not found"
+                    fi'''
+            }
+        }
+
+
+        stage('Checkout') {
+            steps {
+                checkout scm
+            }
+        }
+
+        stage('Compile') {
+            steps {
+                sh './gradlew clean processSpring compileJava compileTestJava --no-daemon --console=plain'
+            }
+        }
+
+        stage('Tests') {
+            parallel {
+                stage('Other Tests') {
+                    stages {
+                        stage('Unit-Tests') {
+                            steps {
+                                sh './gradlew unitTest --no-daemon --console=plain'
+                            }
+                        }
+                        stage('Migration-Tests') {
+                            steps {
+                                sh './gradlew migrationTest --no-daemon --console=plain'
+                            }
+                        }
+                        stage('Scenario-Tests') {
+                            steps {
+                                sh './gradlew scenarioTest --no-daemon --console=plain'
+                            }
+                        }
+                        stage('General-Tests') {
+                            when {
+                                expression { !params.QUICK_RUN }
+                            }
+                            steps {
+                                sh './gradlew generalIntegrationTest --no-daemon --console=plain'
+                            }
+                        }
+                        stage('Booking+Hosting-Tests') {
+                            when {
+                                expression { !params.QUICK_RUN}
+                            }
+                            steps {
+                                sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon --console=plain'
+                            }
+                        }
+                    }
+                }
+
+                // in parallel because these tests take about as much time as all others combined
+                stage('Office-Tests') {
+                    when {
+                        expression { !params.QUICK_RUN }
+                    }
+                    steps {
+                        sh './gradlew officeIntegrationTest --no-daemon --console=plain --fail-fast'
+                    }
+                }
+
+            }
+        }
+
+        stage ('Checks') {
+            steps {
+                sh './gradlew check -x pitest -x test -x dependencyCheckAnalyze --no-daemon'
+            }
+        }
+    }
+
+    post {
+        always {
+            // archive test results
+            junit testResults: 'build/test-results/*/*.xml', allowEmptyResults: true, checksName: '', skipPublishingChecks: true
+
+            // archive the JaCoCo coverage report
+            // recordCoverage tools: [jacoco(pattern: 'build/reports/jacoco/test/jacocoTestReport.xml')]
+            sh 'find build -name jacocoTestReport.xml'
+            archiveArtifacts artifacts: 'build/reports/jacoco/**/jacocoTestReport.xml', allowEmptyArchive: true
+
+            // archive scenario-test reports in HTML format
+            sh './gradlew convertMarkdownToHtml'
+            archiveArtifacts artifacts:
+                    'build/doc/scenarios/*.html, ' +
+                    'build/reports/dependency-license/dependencies-without-allowed-license.json',
+                allowEmptyArchive: true
+
+            // cleanup workspace
+            cleanWs()
+        }
+    }
+}

Jenkins/Makefile

@@ -0,0 +1,54 @@
+DOCKER := docker
+SOCKET := /var/run/docker.sock
+VOLUME := jenkins_home
+
+.PHONY: build run bash init-pw unprotected protected start stop rm purge
+
+# building the Jenkins image
+build:
+	$(DOCKER) build -t jenkins-docker .
+
+# initially running the Jenkins container
+run:
+	$(DOCKER) run --detach \
+		--dns 8.8.8.8 \
+		--network bridge \
+		--publish 8080:8080 --publish 50000:50000 \
+		--volume $(SOCKET):/var/run/docker.sock \
+		--volume $(VOLUME):/var/jenkins_home \
+		--restart unless-stopped \
+		--name jenkins jenkins-docker
+
+# (re-) starts the Jenkins container
+start:
+	$(DOCKER) start jenkins
+
+# opens a bash within the Jenkins container
+bash:
+	$(DOCKER) exec -it jenkins bash
+
+# prints the inital password of a newly setup Jenkins
+init-pw:
+	$(DOCKER) exec -it jenkins cat /var/jenkins_home/secrets/initialAdminPassword
+
+# disables security for the Jenkins, allows login without credentials
+unprotected:
+	docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>false</useSecurity>|' /var/jenkins_home/config.xml
+	docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
+
+# enables security for the Jenkins, requires login with credentials
+protected:
+	docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>true</useSecurity>|' /var/jenkins_home/config.xml
+	docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
+
+# stops the Jenkins container
+stop:
+	$(DOCKER) stop jenkins
+
+# removes the Jenkins container
+rm: stop
+	$(DOCKER) rm jenkins
+
+# purges the Jenkins volume (finally deletes the configuration)
+purge: rm
+	$(DOCKER) volume rm $(VOLUME)

Jenkins/agent/Dockerfile

@@ -0,0 +1,9 @@
+FROM eclipse-temurin:21-jdk
+RUN apt-get update && \
+    apt-get install -y \
+        postgresql-client \
+        bind9-utils \
+        docker.io \
+        pandoc && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*

Jenkinsfile

@@ -1,103 +0,0 @@
-pipeline {
-    agent {
-        dockerfile {
-            filename 'etc/jenkinsAgent.Dockerfile'
-            // additionalBuildArgs  ...
-            args '--network=bridge --user root -v $PWD:$PWD \
-                    -v /var/run/docker.sock:/var/run/docker.sock --group-add 984 \
-                    --memory=6g --cpus=3'
-       }
-    }
-
-    environment {
-        DOCKER_HOST = 'unix:///var/run/docker.sock'
-        HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin'
-        HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted'
-        HSADMINNG_MIGRATION_DATA_PATH = 'migration'
-    }
-
-    triggers {
-        pollSCM('H/1 * * * *')
-    }
-
-    stages {
-        stage('Checkout') {
-            steps {
-                checkout scm
-            }
-        }
-
-        stage ('Compile') {
-            steps {
-                sh './gradlew clean processSpring compileJava compileTestJava --no-daemon'
-            }
-        }
-
-        stage ('Tests') {
-            parallel {
-                stage('Unit-Tests') {
-                    steps {
-                        sh './gradlew unitTest  --no-daemon'
-                    }
-                }
-                stage('General-Tests') {
-                    steps {
-                        sh './gradlew generalTest --no-daemon'
-                    }
-                }
-                stage('Office-Tests') {
-                    steps {
-                        sh './gradlew officeIntegrationTest --no-daemon'
-                    }
-                }
-                stage('Booking+Hosting-Tests') {
-                    steps {
-                        sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon'
-                    }
-                }
-                stage('Test-Imports') {
-                    steps {
-                        sh './gradlew importHostingAssets --no-daemon'
-                    }
-                }
-                stage ('Scenario-Tests') {
-                    steps {
-                        sh './gradlew scenarioTest --no-daemon'
-                    }
-                }
-            }
-        }
-
-        stage ('Check') {
-            steps {
-                sh './gradlew check -x pitest -x dependencyCheckAnalyze --no-daemon'
-            }
-        }
-    }
-
-    post {
-        always {
-            // archive test results
-            junit 'build/test-results/test/*.xml'
-
-            // archive the JaCoCo coverage report in XML and HTML format
-            jacoco(
-                execPattern: 'build/jacoco/*.exec',
-                classPattern: 'build/classes/java/main',
-                sourcePattern: 'src/main/java'
-            )
-
-            // archive scenario-test reports in HTML format
-            sh '''
-                ./gradlew convertMarkdownToHtml
-            '''
-            archiveArtifacts artifacts:
-                    'build/doc/scenarios/*.html, ' +
-                    'build/reports/dependency-license/dependencies-without-allowed-license.json',
-                allowEmptyArchive: true
-
-            // cleanup workspace
-            cleanWs()
-        }
-    }
-}

build.gradle.kts

@@ -151,7 +151,7 @@ tasks.named<Test>("test") {
         excludeTestsMatching("net.hostsharing.hsadminng.**.generated.**")
         // Add more exclude patterns if needed
     }
-    finalizedBy(tasks.named("jacocoTestReport")) // generate report after tests
+    finalizedBy(tasks.named("jacocoTestReport")) // generate a report after tests
 }
 
 // OpenAPI Source Code Generation
@@ -425,18 +425,23 @@ tasks.named<JacocoCoverageVerification>("jacocoTestCoverageVerification") {
 tasks.register<Test>("unitTest") {
     useJUnitPlatform {
         excludeTags(
-            "importHostingAssets", "scenarioTest", "generalIntegrationTest",
+            "importHostingAssets", "scenarioTest", "migrationTest", "generalIntegrationTest",
             "officeIntegrationTest", "bookingIntegrationTest", "hostingIntegrationTest"
         )
     }
 
+    testLogging {
+        events("passed", "skipped", "failed", "standardOut", "standardError")
+        showStandardStreams = true
+    }
+
     group = "verification"
     description = "runs all unit-tests which do not need a database"
 
     mustRunAfter(tasks.named("spotlessJava"))
 }
 
-// HOWTO: run all integration tests which are not specific to a module, like base, rbac, config etc.
+// HOWTO: run all integration tests that are not specific to a module, like base, rbac, config etc.
 tasks.register<Test>("generalIntegrationTest") {
     useJUnitPlatform {
         includeTags("generalIntegrationTest")
@@ -484,6 +489,17 @@ tasks.register<Test>("hostingIntegrationTest") {
     mustRunAfter(tasks.named("spotlessJava"))
 }
 
+tasks.register<Test>("migrationTest") {
+    useJUnitPlatform {
+        includeTags("migrationTest")
+    }
+
+    group = "verification"
+    description = "run database migration tests"
+
+    mustRunAfter(tasks.named("spotlessJava"))
+}
+
 tasks.register<Test>("importHostingAssets") {
     useJUnitPlatform {
         includeTags("importHostingAssets")
@@ -574,14 +590,22 @@ tasks.register("convertMarkdownToHtml") {
 
     // Define the template file using project.file
     val templateFile = project.file("doc/scenarios/.template.html")
-    // Define input directory using layout property
-    val inputDir = layout.buildDirectory.dir("doc/scenarios")
-
-    // Use inputs and outputs for better up-to-date checks
     inputs.file(templateFile).withPathSensitivity(PathSensitivity.NONE)
-    inputs.dir(inputDir).withPathSensitivity(PathSensitivity.RELATIVE)
+
+    // Define input+output directory using layout property
+    val inputDir = layout.buildDirectory.dir("doc/scenarios")
     outputs.dir(inputDir) // Output HTMLs will be in the same directory
 
+    onlyIf {
+        val dir = inputDir.get().asFile
+        if (!dir.exists()) {
+            logger.lifecycle("Skipping convertMarkdownToHtml because ${dir} does not exist (scenarioTest skipped).")
+            false
+        } else {
+            true
+        }
+    }
+
     doFirst {
         // Check if pandoc is installed using exec and capturing output/errors
         val result = project.exec {
@@ -598,21 +622,9 @@ tasks.register("convertMarkdownToHtml") {
         if (!templateFile.exists()) {
             throw GradleException("Template file '$templateFile' not found.")
         }
-        // Ensure input directory exists (Gradle handles this implicitly usually, but explicit check is fine)
-        if (!inputDir.get().asFile.exists()) {
-            logger.warn("Input directory ${inputDir.get().asFile} does not exist, skipping Pandoc conversion.")
-            // Potentially disable the task or skip doLast if input dir missing
-            enabled = false // Example: disable task if input dir doesn't exist yet
-        }
     }
 
     doLast {
-        // Check if input dir exists again, in case it was created between doFirst and doLast
-        if (!inputDir.get().asFile.exists()) {
-            logger.warn("Input directory ${inputDir.get().asFile} still does not exist, skipping Pandoc conversion.")
-            return@doLast // Skip execution
-        }
-
         // Gather all Markdown files in the input directory
         project.fileTree(inputDir) {
             include("*.md")

etc/allowed-licenses.json

@@ -21,10 +21,18 @@
             "moduleVersion": "1.0.0",
             "moduleName": "org.jspecify:jspecify"
         },
+        {
+            "moduleLicense": null,
+            "#moduleLicense": "BSD 3-clause",
+            "moduleVersion": "4.13.0",
+            "moduleName": "org.antlr:antlr4-runtime"
+        },
 
         { "moduleLicense": "BSD License" },
         { "moduleLicense": "BSD-2-Clause" },
+        { "moduleLicense": "The 2-Clause BSD License" },
         { "moduleLicense": "BSD-3-Clause" },
+        { "moduleLicense": "The 3-Clause BSD License" },
         { "moduleLicense": "The BSD License" },
 
         { "moduleLicense": "The New BSD License" },
@@ -45,6 +53,8 @@
 
         { "moduleLicense": "GNU Library General Public License v2.1 or later" },
         { "moduleLicense": "GNU General Public License, version 2 with the GNU Classpath Exception" },
+        { "moduleLicense": "GNU LESSER GENERAL PUBLIC LICENSE, Version 2.1" },
+
         { "moduleLicense": "GPL2 w/ CPE" },
 
         { "moduleLicense": "LGPL, version 2.1"},
@@ -57,11 +67,8 @@
 
         { "moduleLicense": "WTFPL" },
 
-        {
+        { "moduleLicense":  "Creative Commons Legal Code" },
-            "moduleLicense": "Public Domain, per Creative Commons CC0",
+        { "moduleLicense":  "PUBLIC DOMAIN" },
-            "moduleVersion": "2.0.3"
+        { "moduleLicense": "Public Domain, per Creative Commons CC0" }
-        }
-
-
     ]
 }

etc/jenkinsAgent.Dockerfile

@@ -1,6 +0,0 @@
-FROM eclipse-temurin:21-jdk
-RUN apt-get update && \
-    apt-get install -y bind9-utils pandoc && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-

src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java

@@ -89,6 +89,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assumptions.assumeThat;
 import static org.springframework.util.FileCopyUtils.copyToByteArray;
 
+@Tag("migrationTest")
 @Tag("importHostingAssets")
 @DataJpaTest(properties = {
         "spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}",

src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java

@@ -34,7 +34,7 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
  * <p>During a release, the generated dump has to be committed to git and will be used in future test-runs
  * until it gets replaced with a new dump at the next release.</p>
  */
-@Tag("officeIntegrationTest")
+@Tag("migrationTest")
 @DataJpaTest(properties = {
         "spring.datasource.url=jdbc:tc:postgresql:15.5-bookworm:///liquibaseMigrationTestTC",
         "hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",

src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java

@@ -46,6 +46,7 @@ import static org.junit.platform.commons.util.StringUtils.isNotBlank;
 public abstract class UseCase<T extends UseCase<?>> {
 
     private static final HttpClient client = HttpClient.newHttpClient();
+    private static final int HTTP_TIMEOUT_SECONDS = 20; // FIXME: configurable in environment
     private final ObjectMapper objectMapper = new ObjectMapper();
 
     protected final ScenarioTest testSuite;
@@ -160,7 +161,7 @@ public abstract class UseCase<T extends UseCase<?>> {
                 .GET()
                 .uri(new URI("http://localhost:" + testSuite.port + uriPath))
                 .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
-                .timeout(seconds(10))
+                .timeout(seconds(HTTP_TIMEOUT_SECONDS))
                 .build();
         final var response = client.send(request, BodyHandlers.ofString());
         return new HttpResponse(HttpMethod.GET, uriPath, null, response);
@@ -175,7 +176,7 @@ public abstract class UseCase<T extends UseCase<?>> {
                 .uri(new URI("http://localhost:" + testSuite.port + uriPath))
                 .header("Content-Type", "application/json")
                 .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
-                .timeout(seconds(10))
+                .timeout(seconds(HTTP_TIMEOUT_SECONDS))
                 .build();
         final var response = client.send(request, BodyHandlers.ofString());
         return new HttpResponse(HttpMethod.POST, uriPath, requestBody, response);
@@ -190,7 +191,7 @@ public abstract class UseCase<T extends UseCase<?>> {
                 .uri(new URI("http://localhost:" + testSuite.port + uriPath))
                 .header("Content-Type", "application/json")
                 .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
-                .timeout(seconds(10))
+                .timeout(seconds(HTTP_TIMEOUT_SECONDS))
                 .build();
         final var response = client.send(request, BodyHandlers.ofString());
         return new HttpResponse(HttpMethod.PATCH, uriPath, requestBody, response);
@@ -204,7 +205,7 @@ public abstract class UseCase<T extends UseCase<?>> {
                 .uri(new URI("http://localhost:" + testSuite.port + uriPath))
                 .header("Content-Type", "application/json")
                 .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
-                .timeout(seconds(10))
+                .timeout(seconds(HTTP_TIMEOUT_SECONDS))
                 .build();
         final var response = client.send(request, BodyHandlers.ofString());
         return new HttpResponse(HttpMethod.DELETE, uriPath, null, response);

src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java

@@ -5,6 +5,7 @@ import net.hostsharing.hsadminng.mapper.Array;
 import net.hostsharing.hsadminng.mapper.StrictMapper;
 import net.hostsharing.hsadminng.persistence.EntityManagerWrapper;
 import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -41,6 +42,11 @@ class ContextIntegrationTests {
     @PersistenceContext
     private EntityManager em;
 
+    @BeforeAll
+    static void disableRyuk() {
+        System.setProperty("testcontainers.ryuk.disabled", "true");
+    }
+
     @Test
     void defineWithoutHttpServletRequestUsesCallStack() {