hostsharing/hs.hsadmin.ng
1
0
Fork 0
Code Activity

containerized Jenkins (#179)

Browse Source 
Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: https://dev.hostsharing.net/hostsharing/hs.hsadmin.ng/pulls/179
Reviewed-by: Timotheus Pokorra <timotheus.pokorra@hostsharing.net>
This commit is contained in:
Michael Hoennig
2025-06-18 13:51:38 +02:00
parent d351c9a2c1
commit ad1537b856
14 changed files with 283 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.aliases
View File

@@ -129,8 +129,7 @@ function _gwTest() {
# delierately in separate gradlew-calls to avoid Testcontains-PostgreSQL problem spillover
time (_gwTest1 unitTest "$@" &&
_gwTest1 officeIntegrationTest bookingIntegrationTest hostingIntegrationTest "$@" &&
_gwTest1 scenarioTest "$@" &&
_gwTest1 importHostingAssets "$@");
_gwTest1 scenarioTest "$@" && _gwTest1 migrationTest "$@");
elif [ $# -eq 0 ] || [[ $1 == -* ]]; then
time _gwTest1 test "$@";
else

15
Jenkins/Dockerfile Normal file
View File

@@ -0,0 +1,15 @@
FROM jenkins/jenkins:lts-jdk21
USER root
# Docker CLI installieren
RUN apt-get update && apt-get install -y docker.io && usermod -aG docker jenkins
# grant user jenkins access to /var/run/docker.sock
RUN usermod -aG messagebus jenkins
# install plugins
COPY Jenkins.plugins /usr/share/jenkins/ref/plugins.txt
RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt
USER jenkins

8
Jenkins/Jenkins.plugins Normal file
View File

@@ -0,0 +1,8 @@
git
workflow-aggregator
pipeline-github-lib
docker-workflow
credentials
git-client
blueocean
coverage

138
Jenkins/Jenkinsfile vendored Normal file
View File

@@ -0,0 +1,138 @@
pipeline {
parameters {
string(name: 'AGENT_CPUS', defaultValue: '2.5', description: 'CPU limit for the build agent')
string(name: 'AGENT_NETWORK', defaultValue: 'host', description: 'Network to be used for build agent')
booleanParam(name: 'QUICK_RUN', defaultValue: false, description: 'false: all stages but slow, true: just some stages and fast')
}
agent {
dockerfile {
filename 'Jenkins/agent/Dockerfile'
args """--user root --network ${params.AGENT_NETWORK}
--volume /var/run/docker.sock:/var/run/docker.sock
--memory=8g --cpus=${params.AGENT_CPUS}"""
}
}
environment {
GRADLE_USER_HOME = "${env.WORKSPACE}/.gradle-cache"
DOCKER_HOST = 'unix:///var/run/docker.sock'
HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin'
HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted'
HSADMINNG_MIGRATION_DATA_PATH = 'migration'
TESTCONTAINERS_RYUK_DISABLED = true
TESTCONTAINERS_LOG_LEVEL = "DEBUG"
}
triggers {
pollSCM('H/1 * * * *')
}
stages {
stage('Detect Docker Environment') {
steps {
sh '''#!/bin/bash +x
if command -v docker >/dev/null 2>&1; then
if docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'rootless'; then
echo "🟡 Docker daemon is running in ROOTLESS mode"
else
echo "🟢 Docker daemon is running in ROOTFUL mode"
fi
else
echo "❌ Docker CLI not found"
fi'''
}
}
stage('Checkout') {
steps {
checkout scm
}
}
stage('Compile') {
steps {
sh './gradlew clean processSpring compileJava compileTestJava --no-daemon --console=plain'
}
}
stage('Tests') {
parallel {
stage('Other Tests') {
stages {
stage('Unit-Tests') {
steps {
sh './gradlew unitTest --no-daemon --console=plain'
}
}
stage('Migration-Tests') {
steps {
sh './gradlew migrationTest --no-daemon --console=plain'
}
}
stage('Scenario-Tests') {
steps {
sh './gradlew scenarioTest --no-daemon --console=plain'
}
}
stage('General-Tests') {
when {
expression { !params.QUICK_RUN }
}
steps {
sh './gradlew generalIntegrationTest --no-daemon --console=plain'
}
}
stage('Booking+Hosting-Tests') {
when {
expression { !params.QUICK_RUN}
}
steps {
sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon --console=plain'
}
}
}
}
// in parallel because these tests take about as much time as all others combined
stage('Office-Tests') {
when {
expression { !params.QUICK_RUN }
}
steps {
sh './gradlew officeIntegrationTest --no-daemon --console=plain --fail-fast'
}
}
}
}
stage ('Checks') {
steps {
sh './gradlew check -x pitest -x test -x dependencyCheckAnalyze --no-daemon'
}
}
}
post {
always {
// archive test results
junit testResults: 'build/test-results/*/*.xml', allowEmptyResults: true, checksName: '', skipPublishingChecks: true
// archive the JaCoCo coverage report
// recordCoverage tools: [jacoco(pattern: 'build/reports/jacoco/test/jacocoTestReport.xml')]
sh 'find build -name jacocoTestReport.xml'
archiveArtifacts artifacts: 'build/reports/jacoco/**/jacocoTestReport.xml', allowEmptyArchive: true
// archive scenario-test reports in HTML format
sh './gradlew convertMarkdownToHtml'
archiveArtifacts artifacts:
'build/doc/scenarios/*.html, ' +
'build/reports/dependency-license/dependencies-without-allowed-license.json',
allowEmptyArchive: true
// cleanup workspace
cleanWs()
}
}
}

54
Jenkins/Makefile Normal file
View File

@@ -0,0 +1,54 @@
DOCKER := docker
SOCKET := /var/run/docker.sock
VOLUME := jenkins_home
.PHONY: build run bash init-pw unprotected protected start stop rm purge
# building the Jenkins image
build:
$(DOCKER) build -t jenkins-docker .
# initially running the Jenkins container
run:
$(DOCKER) run --detach \
--dns 8.8.8.8 \
--network bridge \
--publish 8080:8080 --publish 50000:50000 \
--volume $(SOCKET):/var/run/docker.sock \
--volume $(VOLUME):/var/jenkins_home \
--restart unless-stopped \
--name jenkins jenkins-docker
# (re-) starts the Jenkins container
start:
$(DOCKER) start jenkins
# opens a bash within the Jenkins container
bash:
$(DOCKER) exec -it jenkins bash
# prints the inital password of a newly setup Jenkins
init-pw:
$(DOCKER) exec -it jenkins cat /var/jenkins_home/secrets/initialAdminPassword
# disables security for the Jenkins, allows login without credentials
unprotected:
docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>false</useSecurity>|' /var/jenkins_home/config.xml
docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
# enables security for the Jenkins, requires login with credentials
protected:
docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>true</useSecurity>|' /var/jenkins_home/config.xml
docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
# stops the Jenkins container
stop:
$(DOCKER) stop jenkins
# removes the Jenkins container
rm: stop
$(DOCKER) rm jenkins
# purges the Jenkins volume (finally deletes the configuration)
purge: rm
$(DOCKER) volume rm $(VOLUME)

9
Jenkins/agent/Dockerfile Normal file
View File

@@ -0,0 +1,9 @@
FROM eclipse-temurin:21-jdk
RUN apt-get update && \
apt-get install -y \
postgresql-client \
bind9-utils \
docker.io \
pandoc && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

103
Jenkinsfile vendored
View File

@@ -1,103 +0,0 @@
pipeline {
agent {
dockerfile {
filename 'etc/jenkinsAgent.Dockerfile'
// additionalBuildArgs ...
args '--network=bridge --user root -v $PWD:$PWD \
-v /var/run/docker.sock:/var/run/docker.sock --group-add 984 \
--memory=6g --cpus=3'
}
}
environment {
DOCKER_HOST = 'unix:///var/run/docker.sock'
HSADMINNG_POSTGRES_ADMIN_USERNAME = 'admin'
HSADMINNG_POSTGRES_RESTRICTED_USERNAME = 'restricted'
HSADMINNG_MIGRATION_DATA_PATH = 'migration'
}
triggers {
pollSCM('H/1 * * * *')
}
stages {
stage('Checkout') {
steps {
checkout scm
}
}
stage ('Compile') {
steps {
sh './gradlew clean processSpring compileJava compileTestJava --no-daemon'
}
}
stage ('Tests') {
parallel {
stage('Unit-Tests') {
steps {
sh './gradlew unitTest --no-daemon'
}
}
stage('General-Tests') {
steps {
sh './gradlew generalTest --no-daemon'
}
}
stage('Office-Tests') {
steps {
sh './gradlew officeIntegrationTest --no-daemon'
}
}
stage('Booking+Hosting-Tests') {
steps {
sh './gradlew bookingIntegrationTest hostingIntegrationTest --no-daemon'
}
}
stage('Test-Imports') {
steps {
sh './gradlew importHostingAssets --no-daemon'
}
}
stage ('Scenario-Tests') {
steps {
sh './gradlew scenarioTest --no-daemon'
}
}
}
}
stage ('Check') {
steps {
sh './gradlew check -x pitest -x dependencyCheckAnalyze --no-daemon'
}
}
}
post {
always {
// archive test results
junit 'build/test-results/test/*.xml'
// archive the JaCoCo coverage report in XML and HTML format
jacoco(
execPattern: 'build/jacoco/*.exec',
classPattern: 'build/classes/java/main',
sourcePattern: 'src/main/java'
)
// archive scenario-test reports in HTML format
sh '''
./gradlew convertMarkdownToHtml
'''
archiveArtifacts artifacts:
'build/doc/scenarios/*.html, ' +
'build/reports/dependency-license/dependencies-without-allowed-license.json',
allowEmptyArchive: true
// cleanup workspace
cleanWs()
}
}
}

52
build.gradle.kts
View File

@@ -151,7 +151,7 @@ tasks.named<Test>("test") {
excludeTestsMatching("net.hostsharing.hsadminng.**.generated.**")
// Add more exclude patterns if needed
}
finalizedBy(tasks.named("jacocoTestReport")) // generate report after tests
finalizedBy(tasks.named("jacocoTestReport")) // generate a report after tests
}
// OpenAPI Source Code Generation
@@ -425,18 +425,23 @@ tasks.named<JacocoCoverageVerification>("jacocoTestCoverageVerification") {
tasks.register<Test>("unitTest") {
useJUnitPlatform {
excludeTags(
"importHostingAssets", "scenarioTest", "generalIntegrationTest",
"importHostingAssets", "scenarioTest", "migrationTest", "generalIntegrationTest",
"officeIntegrationTest", "bookingIntegrationTest", "hostingIntegrationTest"
)
}
testLogging {
events("passed", "skipped", "failed", "standardOut", "standardError")
showStandardStreams = true
}
group = "verification"
description = "runs all unit-tests which do not need a database"
mustRunAfter(tasks.named("spotlessJava"))
}
// HOWTO: run all integration tests which are not specific to a module, like base, rbac, config etc.
// HOWTO: run all integration tests that are not specific to a module, like base, rbac, config etc.
tasks.register<Test>("generalIntegrationTest") {
useJUnitPlatform {
includeTags("generalIntegrationTest")
@@ -484,6 +489,17 @@ tasks.register<Test>("hostingIntegrationTest") {
mustRunAfter(tasks.named("spotlessJava"))
}
tasks.register<Test>("migrationTest") {
useJUnitPlatform {
includeTags("migrationTest")
}
group = "verification"
description = "run database migration tests"
mustRunAfter(tasks.named("spotlessJava"))
}
tasks.register<Test>("importHostingAssets") {
useJUnitPlatform {
includeTags("importHostingAssets")
@@ -574,14 +590,22 @@ tasks.register("convertMarkdownToHtml") {
// Define the template file using project.file
val templateFile = project.file("doc/scenarios/.template.html")
// Define input directory using layout property
val inputDir = layout.buildDirectory.dir("doc/scenarios")
// Use inputs and outputs for better up-to-date checks
inputs.file(templateFile).withPathSensitivity(PathSensitivity.NONE)
inputs.dir(inputDir).withPathSensitivity(PathSensitivity.RELATIVE)
// Define input+output directory using layout property
val inputDir = layout.buildDirectory.dir("doc/scenarios")
outputs.dir(inputDir) // Output HTMLs will be in the same directory
onlyIf {
val dir = inputDir.get().asFile
if (!dir.exists()) {
logger.lifecycle("Skipping convertMarkdownToHtml because ${dir} does not exist (scenarioTest skipped).")
false
} else {
true
}
}
doFirst {
// Check if pandoc is installed using exec and capturing output/errors
val result = project.exec {
@@ -598,21 +622,9 @@ tasks.register("convertMarkdownToHtml") {
if (!templateFile.exists()) {
throw GradleException("Template file '$templateFile' not found.")
}
// Ensure input directory exists (Gradle handles this implicitly usually, but explicit check is fine)
if (!inputDir.get().asFile.exists()) {
logger.warn("Input directory ${inputDir.get().asFile} does not exist, skipping Pandoc conversion.")
// Potentially disable the task or skip doLast if input dir missing
enabled = false // Example: disable task if input dir doesn't exist yet
}
}
doLast {
// Check if input dir exists again, in case it was created between doFirst and doLast
if (!inputDir.get().asFile.exists()) {
logger.warn("Input directory ${inputDir.get().asFile} still does not exist, skipping Pandoc conversion.")
return@doLast // Skip execution
}
// Gather all Markdown files in the input directory
project.fileTree(inputDir) {
include("*.md")

19
etc/allowed-licenses.json
View File

@@ -21,10 +21,18 @@
"moduleVersion": "1.0.0",
"moduleName": "org.jspecify:jspecify"
},
{
"moduleLicense": null,
"#moduleLicense": "BSD 3-clause",
"moduleVersion": "4.13.0",
"moduleName": "org.antlr:antlr4-runtime"
},
{ "moduleLicense": "BSD License" },
{ "moduleLicense": "BSD-2-Clause" },
{ "moduleLicense": "The 2-Clause BSD License" },
{ "moduleLicense": "BSD-3-Clause" },
{ "moduleLicense": "The 3-Clause BSD License" },
{ "moduleLicense": "The BSD License" },
{ "moduleLicense": "The New BSD License" },
@@ -45,6 +53,8 @@
{ "moduleLicense": "GNU Library General Public License v2.1 or later" },
{ "moduleLicense": "GNU General Public License, version 2 with the GNU Classpath Exception" },
{ "moduleLicense": "GNU LESSER GENERAL PUBLIC LICENSE, Version 2.1" },
{ "moduleLicense": "GPL2 w/ CPE" },
{ "moduleLicense": "LGPL, version 2.1"},
@@ -57,11 +67,8 @@
{ "moduleLicense": "WTFPL" },
{
"moduleLicense": "Public Domain, per Creative Commons CC0",
"moduleVersion": "2.0.3"
}
{ "moduleLicense": "Creative Commons Legal Code" },
{ "moduleLicense": "PUBLIC DOMAIN" },
{ "moduleLicense": "Public Domain, per Creative Commons CC0" }
]
}

6
etc/jenkinsAgent.Dockerfile
View File

@@ -1,6 +0,0 @@
FROM eclipse-temurin:21-jdk
RUN apt-get update && \
apt-get install -y bind9-utils pandoc && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

1
src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java
View File

@@ -89,6 +89,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assumptions.assumeThat;
import static org.springframework.util.FileCopyUtils.copyToByteArray;
@Tag("migrationTest")
@Tag("importHostingAssets")
@DataJpaTest(properties = {
"spring.datasource.url=${HSADMINNG_POSTGRES_JDBC_URL:jdbc:tc:postgresql:15.5-bookworm:///importHostingAssetsTC}",

2
src/test/java/net/hostsharing/hsadminng/hs/migration/LiquibaseCompatibilityIntegrationTest.java
View File

@@ -34,7 +34,7 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
* <p>During a release, the generated dump has to be committed to git and will be used in future test-runs
* until it gets replaced with a new dump at the next release.</p>
*/
@Tag("officeIntegrationTest")
@Tag("migrationTest")
@DataJpaTest(properties = {
"spring.datasource.url=jdbc:tc:postgresql:15.5-bookworm:///liquibaseMigrationTestTC",
"hsadminng.superuser=${HSADMINNG_SUPERUSER:import-superuser@hostsharing.net}",

9
src/test/java/net/hostsharing/hsadminng/hs/scenarios/UseCase.java
View File

@@ -46,6 +46,7 @@ import static org.junit.platform.commons.util.StringUtils.isNotBlank;
public abstract class UseCase<T extends UseCase<?>> {
private static final HttpClient client = HttpClient.newHttpClient();
private static final int HTTP_TIMEOUT_SECONDS = 20; // FIXME: configurable in environment
private final ObjectMapper objectMapper = new ObjectMapper();
protected final ScenarioTest testSuite;
@@ -160,7 +161,7 @@ public abstract class UseCase<T extends UseCase<?>> {
.GET()
.uri(new URI("http://localhost:" + testSuite.port + uriPath))
.header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
.timeout(seconds(10))
.timeout(seconds(HTTP_TIMEOUT_SECONDS))
.build();
final var response = client.send(request, BodyHandlers.ofString());
return new HttpResponse(HttpMethod.GET, uriPath, null, response);
@@ -175,7 +176,7 @@ public abstract class UseCase<T extends UseCase<?>> {
.uri(new URI("http://localhost:" + testSuite.port + uriPath))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
.timeout(seconds(10))
.timeout(seconds(HTTP_TIMEOUT_SECONDS))
.build();
final var response = client.send(request, BodyHandlers.ofString());
return new HttpResponse(HttpMethod.POST, uriPath, requestBody, response);
@@ -190,7 +191,7 @@ public abstract class UseCase<T extends UseCase<?>> {
.uri(new URI("http://localhost:" + testSuite.port + uriPath))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
.timeout(seconds(10))
.timeout(seconds(HTTP_TIMEOUT_SECONDS))
.build();
final var response = client.send(request, BodyHandlers.ofString());
return new HttpResponse(HttpMethod.PATCH, uriPath, requestBody, response);
@@ -204,7 +205,7 @@ public abstract class UseCase<T extends UseCase<?>> {
.uri(new URI("http://localhost:" + testSuite.port + uriPath))
.header("Content-Type", "application/json")
.header("Authorization", "Bearer " + ScenarioTest.RUN_AS_USER)
.timeout(seconds(10))
.timeout(seconds(HTTP_TIMEOUT_SECONDS))
.build();
final var response = client.send(request, BodyHandlers.ofString());
return new HttpResponse(HttpMethod.DELETE, uriPath, null, response);

6
src/test/java/net/hostsharing/hsadminng/rbac/context/ContextIntegrationTests.java
View File

@@ -5,6 +5,7 @@ import net.hostsharing.hsadminng.mapper.Array;
import net.hostsharing.hsadminng.mapper.StrictMapper;
import net.hostsharing.hsadminng.persistence.EntityManagerWrapper;
import net.hostsharing.hsadminng.rbac.test.JpaAttempt;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -41,6 +42,11 @@ class ContextIntegrationTests {
@PersistenceContext
private EntityManager em;
@BeforeAll
static void disableRyuk() {
System.setProperty("testcontainers.ryuk.disabled", "true");
}
@Test
void defineWithoutHttpServletRequestUsesCallStack() {