diff --git a/.aliases b/.aliases index 5da642ab..2e72272f 100644 --- a/.aliases +++ b/.aliases @@ -129,8 +129,7 @@ function _gwTest() { # delierately in separate gradlew-calls to avoid Testcontains-PostgreSQL problem spillover time (_gwTest1 unitTest "$@" && _gwTest1 officeIntegrationTest bookingIntegrationTest hostingIntegrationTest "$@" && - _gwTest1 scenarioTest "$@" && - _gwTest1 importHostingAssets "$@"); + _gwTest1 scenarioTest "$@" && _gwTest1 migrationTest "$@"); elif [ $# -eq 0 ] || [[ $1 == -* ]]; then time _gwTest1 test "$@"; else diff --git a/Jenkins/Dockerfile b/Jenkins/Dockerfile new file mode 100644 index 00000000..e4f70c9a --- /dev/null +++ b/Jenkins/Dockerfile @@ -0,0 +1,15 @@ +FROM jenkins/jenkins:lts-jdk21 + +USER root + +# Docker CLI installieren +RUN apt-get update && apt-get install -y docker.io && usermod -aG docker jenkins + +# grant user jenkins access to /var/run/docker.sock +RUN usermod -aG messagebus jenkins + +# install plugins +COPY Jenkins.plugins /usr/share/jenkins/ref/plugins.txt +RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt + +USER jenkins diff --git a/Jenkins/Jenkins.plugins b/Jenkins/Jenkins.plugins new file mode 100644 index 00000000..a8cb2f88 --- /dev/null +++ b/Jenkins/Jenkins.plugins @@ -0,0 +1,8 @@ +git +workflow-aggregator +pipeline-github-lib +docker-workflow +credentials +git-client +blueocean +coverage diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile new file mode 100644 index 00000000..34332d46 --- /dev/null +++ b/Jenkins/Jenkinsfile @@ -0,0 +1,138 @@ +pipeline { + parameters { + string(name: 'AGENT_CPUS', defaultValue: '2.5', description: 'CPU limit for the build agent') + string(name: 'AGENT_NETWORK', defaultValue: 'host', description: 'Network to be used for build agent') + booleanParam(name: 'QUICK_RUN', defaultValue: false, description: 'false: all stages but slow, true: just some stages and fast') + } + agent { + dockerfile { + filename 'Jenkins/agent/Dockerfile' + args """--user root --network ${params.AGENT_NETWORK} + --volume /var/run/docker.sock:/var/run/docker.sock + --memory=8g --cpus=${params.AGENT_CPUS}""" + } + } + + environment { + GRADLE_USER_HOME = "${env.WORKSPACE}/.gradle-cache" + DOCKER_HOST = 'unix:///var/run/docker.sock' + HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin' + HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted' + HSADMINNG_MIGRATION_DATA_PATH = 'migration' + TESTCONTAINERS_RYUK_DISABLED = true + TESTCONTAINERS_LOG_LEVEL = "DEBUG" + } + + triggers { + pollSCM('H/1 * * * *') + } + + stages { + stage('Detect Docker Environment') { + steps { + sh '''#!/bin/bash +x + if command -v docker >/dev/null 2>&1; then + if docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'rootless'; then + echo "🟡 Docker daemon is running in ROOTLESS mode" + else + echo "🟢 Docker daemon is running in ROOTFUL mode" + fi + else + echo "❌ Docker CLI not found" + fi''' + } + } + + + stage('Checkout') { + steps { + checkout scm + } + } + + stage('Compile') { + steps { + sh './gradlew clean processSpring compileJava compileTestJava --no-daemon --console=plain' + } + } + + stage('Tests') { + parallel { + stage('Other Tests') { + stages { + stage('Unit-Tests') { + steps { + sh './gradlew unitTest --no-daemon --console=plain' + } + } + stage('Migration-Tests') { + steps { + sh './gradlew migrationTest --no-daemon --console=plain' + } + } + stage('Scenario-Tests') { + steps { + sh './gradlew scenarioTest --no-daemon --console=plain' + } + } + stage('General-Tests') { + when { + expression { !params.QUICK_RUN } + } + steps { + sh './gradlew generalIntegrationTest --no-daemon --console=plain' + } + } + stage('Booking+Hosting-Tests') { + when { + expression { !params.QUICK_RUN} + } + steps { + sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon --console=plain' + } + } + } + } + + // in parallel because these tests take about as much time as all others combined + stage('Office-Tests') { + when { + expression { !params.QUICK_RUN } + } + steps { + sh './gradlew officeIntegrationTest --no-daemon --console=plain --fail-fast' + } + } + + } + } + + stage ('Checks') { + steps { + sh './gradlew check -x pitest -x test -x dependencyCheckAnalyze --no-daemon' + } + } + } + + post { + always { + // archive test results + junit testResults: 'build/test-results/*/*.xml', allowEmptyResults: true, checksName: '', skipPublishingChecks: true + + // archive the JaCoCo coverage report + // recordCoverage tools: [jacoco(pattern: 'build/reports/jacoco/test/jacocoTestReport.xml')] + sh 'find build -name jacocoTestReport.xml' + archiveArtifacts artifacts: 'build/reports/jacoco/**/jacocoTestReport.xml', allowEmptyArchive: true + + // archive scenario-test reports in HTML format + sh './gradlew convertMarkdownToHtml' + archiveArtifacts artifacts: + 'build/doc/scenarios/*.html, ' + + 'build/reports/dependency-license/dependencies-without-allowed-license.json', + allowEmptyArchive: true + + // cleanup workspace + cleanWs() + } + } +} diff --git a/Jenkins/Makefile b/Jenkins/Makefile new file mode 100644 index 00000000..4beb5ebd --- /dev/null +++ b/Jenkins/Makefile @@ -0,0 +1,54 @@ +DOCKER := docker +SOCKET := /var/run/docker.sock +VOLUME := jenkins_home + +.PHONY: build run bash init-pw unprotected protected start stop rm purge + +# building the Jenkins image +build: + $(DOCKER) build -t jenkins-docker . + +# initially running the Jenkins container +run: + $(DOCKER) run --detach \ + --dns 8.8.8.8 \ + --network bridge \ + --publish 8080:8080 --publish 50000:50000 \ + --volume $(SOCKET):/var/run/docker.sock \ + --volume $(VOLUME):/var/jenkins_home \ + --restart unless-stopped \ + --name jenkins jenkins-docker + +# (re-) starts the Jenkins container +start: + $(DOCKER) start jenkins + +# opens a bash within the Jenkins container +bash: + $(DOCKER) exec -it jenkins bash + +# prints the inital password of a newly setup Jenkins +init-pw: + $(DOCKER) exec -it jenkins cat /var/jenkins_home/secrets/initialAdminPassword + +# disables security for the Jenkins, allows login without credentials +unprotected: + docker exec -it jenkins sed -i 's|true|false|' /var/jenkins_home/config.xml + docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml + +# enables security for the Jenkins, requires login with credentials +protected: + docker exec -it jenkins sed -i 's|true|true|' /var/jenkins_home/config.xml + docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml + +# stops the Jenkins container +stop: + $(DOCKER) stop jenkins + +# removes the Jenkins container +rm: stop + $(DOCKER) rm jenkins + +# purges the Jenkins volume (finally deletes the configuration) +purge: rm + $(DOCKER) volume rm $(VOLUME) diff --git a/Jenkins/agent/Dockerfile b/Jenkins/agent/Dockerfile new file mode 100644 index 00000000..e0d571f8 --- /dev/null +++ b/Jenkins/agent/Dockerfile @@ -0,0 +1,9 @@ +FROM eclipse-temurin:21-jdk +RUN apt-get update && \ + apt-get install -y \ + postgresql-client \ + bind9-utils \ + docker.io \ + pandoc && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 19bdd93c..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,103 +0,0 @@ -pipeline { - agent { - dockerfile { - filename 'etc/jenkinsAgent.Dockerfile' - // additionalBuildArgs ... - args '--network=bridge --user root -v $PWD:$PWD \ - -v /var/run/docker.sock:/var/run/docker.sock --group-add 984 \ - --memory=6g --cpus=3' - } - } - - environment { - DOCKER_HOST = 'unix:///var/run/docker.sock' - HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin' - HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted' - HSADMINNG_MIGRATION_DATA_PATH = 'migration' - } - - triggers { - pollSCM('H/1 * * * *') - } - - stages { - stage('Checkout') { - steps { - checkout scm - } - } - - stage ('Compile') { - steps { - sh './gradlew clean processSpring compileJava compileTestJava --no-daemon' - } - } - - stage ('Tests') { - parallel { - stage('Unit-Tests') { - steps { - sh './gradlew unitTest --no-daemon' - } - } - stage('General-Tests') { - steps { - sh './gradlew generalTest --no-daemon' - } - } - stage('Office-Tests') { - steps { - sh './gradlew officeIntegrationTest --no-daemon' - } - } - stage('Booking+Hosting-Tests') { - steps { - sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon' - } - } - stage('Test-Imports') { - steps { - sh './gradlew importHostingAssets --no-daemon' - } - } - stage ('Scenario-Tests') { - steps { - sh './gradlew scenarioTest --no-daemon' - } - } - } - } - - stage ('Check') { - steps { - sh './gradlew check -x pitest -x dependencyCheckAnalyze --no-daemon' - } - } - } - - post { - always { - // archive test results - junit 'build/test-results/test/*.xml' - - // archive the JaCoCo coverage report in XML and HTML format - jacoco( - execPattern: 'build/jacoco/*.exec', - classPattern: 'build/classes/java/main', - sourcePattern: 'src/main/java' - ) - - // archive scenario-test reports in HTML format - sh ''' - ./gradlew convertMarkdownToHtml - ''' - archiveArtifacts artifacts: - 'build/doc/scenarios/*.html, ' + - 'build/reports/dependency-license/dependencies-without-allowed-license.json', - allowEmptyArchive: true - - // cleanup workspace - cleanWs() - } - } -} diff --git a/build.gradle.kts b/build.gradle.kts index cd21c334..fe3e294b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -151,7 +151,7 @@ tasks.named("test") { excludeTestsMatching("net.hostsharing.hsadminng.**.generated.**") // Add more exclude patterns if needed } - finalizedBy(tasks.named("jacocoTestReport")) // generate report after tests + finalizedBy(tasks.named("jacocoTestReport")) // generate a report after tests } // OpenAPI Source Code Generation @@ -425,18 +425,23 @@ tasks.named("jacocoTestCoverageVerification") { tasks.register("unitTest") { useJUnitPlatform { excludeTags( - "importHostingAssets", "scenarioTest", "generalIntegrationTest", + "importHostingAssets", "scenarioTest", "migrationTest", "generalIntegrationTest", "officeIntegrationTest", "bookingIntegrationTest", "hostingIntegrationTest" ) } + testLogging { + events("passed", "skipped", "failed", "standardOut", "standardError") + showStandardStreams = true + } + group = "verification" description = "runs all unit-tests which do not need a database" mustRunAfter(tasks.named("spotlessJava")) } -// HOWTO: run all integration tests which are not specific to a module, like base, rbac, config etc. +// HOWTO: run all integration tests that are not specific to a module, like base, rbac, config etc. tasks.register("generalIntegrationTest") { useJUnitPlatform { includeTags("generalIntegrationTest") @@ -484,6 +489,17 @@ tasks.register("hostingIntegrationTest") { mustRunAfter(tasks.named("spotlessJava")) } +tasks.register("migrationTest") { + useJUnitPlatform { + includeTags("migrationTest") + } + + group = "verification" + description = "run database migration tests" + + mustRunAfter(tasks.named("spotlessJava")) +} + tasks.register("importHostingAssets") { useJUnitPlatform { includeTags("importHostingAssets") @@ -574,14 +590,22 @@ tasks.register("convertMarkdownToHtml") { // Define the template file using project.file val templateFile = project.file("doc/scenarios/.template.html") - // Define input directory using layout property - val inputDir = layout.buildDirectory.dir("doc/scenarios") - - // Use inputs and outputs for better up-to-date checks inputs.file(templateFile).withPathSensitivity(PathSensitivity.NONE) - inputs.dir(inputDir).withPathSensitivity(PathSensitivity.RELATIVE) + + // Define input+output directory using layout property + val inputDir = layout.buildDirectory.dir("doc/scenarios") outputs.dir(inputDir) // Output HTMLs will be in the same directory + onlyIf { + val dir = inputDir.get().asFile + if (!dir.exists()) { + logger.lifecycle("Skipping convertMarkdownToHtml because ${dir} does not exist (scenarioTest skipped).") + false + } else { + true + } + } + doFirst { // Check if pandoc is installed using exec and capturing output/errors val result = project.exec { @@ -598,21 +622,9 @@ tasks.register("convertMarkdownToHtml") { if (!templateFile.exists()) { throw GradleException("Template file '$templateFile' not found.") } - // Ensure input directory exists (Gradle handles this implicitly usually, but explicit check is fine) - if (!inputDir.get().asFile.exists()) { - logger.warn("Input directory ${inputDir.get().asFile} does not exist, skipping Pandoc conversion.") - // Potentially disable the task or skip doLast if input dir missing - enabled = false // Example: disable task if input dir doesn't exist yet - } } doLast { - // Check if input dir exists again, in case it was created between doFirst and doLast - if (!inputDir.get().asFile.exists()) { - logger.warn("Input directory ${inputDir.get().asFile} still does not exist, skipping Pandoc conversion.") - return@doLast // Skip execution - } - // Gather all Markdown files in the input directory project.fileTree(inputDir) { include("*.md") diff --git a/etc/allowed-licenses.json b/etc/allowed-licenses.json index 447f5c62..61ca10e8 100644 --- a/etc/allowed-licenses.json +++ b/etc/allowed-licenses.json @@ -21,10 +21,18 @@ "moduleVersion": "1.0.0", "moduleName": "org.jspecify:jspecify" }, + { + "moduleLicense": null, + "#moduleLicense": "BSD 3-clause", + "moduleVersion": "4.13.0", + "moduleName": "org.antlr:antlr4-runtime" + }, { "moduleLicense": "BSD License" }, { "moduleLicense": "BSD-2-Clause" }, + { "moduleLicense": "The 2-Clause BSD License" }, { "moduleLicense": "BSD-3-Clause" }, + { "moduleLicense": "The 3-Clause BSD License" }, { "moduleLicense": "The BSD License" }, { "moduleLicense": "The New BSD License" }, @@ -45,6 +53,8 @@ { "moduleLicense": "GNU Library General Public License v2.1 or later" }, { "moduleLicense": "GNU General Public License, version 2 with the GNU Classpath Exception" }, + { "moduleLicense": "GNU LESSER GENERAL PUBLIC LICENSE, Version 2.1" }, + { "moduleLicense": "GPL2 w/ CPE" }, { "moduleLicense": "LGPL, version 2.1"}, @@ -57,11 +67,8 @@ { "moduleLicense": "WTFPL" }, - { - "moduleLicense": "Public Domain, per Creative Commons CC0", - "moduleVersion": "2.0.3" - } - - + { "moduleLicense": "Creative Commons Legal Code" }, + { "moduleLicense": "PUBLIC DOMAIN" }, + { "moduleLicense": "Public Domain, per Creative Commons CC0" } ] } diff --git a/etc/jenkinsAgent.Dockerfile b/etc/jenkinsAgent.Dockerfile deleted file mode 100644 index f06e9e4f..00000000 --- a/etc/jenkinsAgent.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM eclipse-temurin:21-jdk -RUN apt-get update && \ - apt-get install -y bind9-utils pandoc && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - diff --git a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java index f1e2bcb8..fc2ff6ca 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java @@ -89,6 +89,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assumptions.assumeThat; import static org.springframework.util.FileCopyUtils.copyToByteArray; +@Tag("migrationTest") @Tag("importHostingAssets") @DataJpaTest(properties = { "spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}", diff --git a/src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java b/src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java index 6667c0ce..e97594bd 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java @@ -34,7 +34,7 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE *

During a release, the generated dump has to be committed to git and will be used in future test-runs * until it gets replaced with a new dump at the next release.

*/ -@Tag("officeIntegrationTest") +@Tag("migrationTest") @DataJpaTest(properties = { "spring.datasource.url=jdbc:tc:postgresql:15.5-bookworm:///liquibaseMigrationTestTC", "hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}", diff --git a/src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java b/src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java index 561c0560..e8cdd946 100644 --- a/src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java +++ b/src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java @@ -46,6 +46,7 @@ import static org.junit.platform.commons.util.StringUtils.isNotBlank; public abstract class UseCase> { private static final HttpClient client = HttpClient.newHttpClient(); + private static final int HTTP_TIMEOUT_SECONDS = 20; // FIXME: configurable in environment private final ObjectMapper objectMapper = new ObjectMapper(); protected final ScenarioTest testSuite; @@ -160,7 +161,7 @@ public abstract class UseCase> { .GET() .uri(new URI("http://localhost:" + testSuite.port + uriPath)) .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER) - .timeout(seconds(10)) + .timeout(seconds(HTTP_TIMEOUT_SECONDS)) .build(); final var response = client.send(request, BodyHandlers.ofString()); return new HttpResponse(HttpMethod.GET, uriPath, null, response); @@ -175,7 +176,7 @@ public abstract class UseCase> { .uri(new URI("http://localhost:" + testSuite.port + uriPath)) .header("Content-Type", "application/json") .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER) - .timeout(seconds(10)) + .timeout(seconds(HTTP_TIMEOUT_SECONDS)) .build(); final var response = client.send(request, BodyHandlers.ofString()); return new HttpResponse(HttpMethod.POST, uriPath, requestBody, response); @@ -190,7 +191,7 @@ public abstract class UseCase> { .uri(new URI("http://localhost:" + testSuite.port + uriPath)) .header("Content-Type", "application/json") .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER) - .timeout(seconds(10)) + .timeout(seconds(HTTP_TIMEOUT_SECONDS)) .build(); final var response = client.send(request, BodyHandlers.ofString()); return new HttpResponse(HttpMethod.PATCH, uriPath, requestBody, response); @@ -204,7 +205,7 @@ public abstract class UseCase> { .uri(new URI("http://localhost:" + testSuite.port + uriPath)) .header("Content-Type", "application/json") .header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER) - .timeout(seconds(10)) + .timeout(seconds(HTTP_TIMEOUT_SECONDS)) .build(); final var response = client.send(request, BodyHandlers.ofString()); return new HttpResponse(HttpMethod.DELETE, uriPath, null, response); diff --git a/src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java b/src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java index f12ea414..c3381f6c 100644 --- a/src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java +++ b/src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java @@ -5,6 +5,7 @@ import net.hostsharing.hsadminng.mapper.Array; import net.hostsharing.hsadminng.mapper.StrictMapper; import net.hostsharing.hsadminng.persistence.EntityManagerWrapper; import net.hostsharing.hsadminng.rbac.test.JpaAttempt; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -41,6 +42,11 @@ class ContextIntegrationTests { @PersistenceContext private EntityManager em; + @BeforeAll + static void disableRyuk() { + System.setProperty("testcontainers.ryuk.disabled", "true"); + } + @Test void defineWithoutHttpServletRequestUsesCallStack() {