make nginx-provisioning work and rename make targets to prefix jenkins-... (#183)

Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: https://dev.hostsharing.net/hostsharing/hs.hsadmin.ng/pulls/183
Reviewed-by: Timotheus Pokorra <timotheus.pokorra@hostsharing.net>

Jenkins/Dockerfile

@@ -5,11 +5,18 @@ USER root
 # Docker CLI installieren
 RUN apt-get update && apt-get install -y docker.io && usermod -aG docker jenkins
 
+# Create workspace directory with correct owner and permissions
+RUN mkdir -p /var/jenkins_home/workspace && \
+    chown -R jenkins:jenkins /var /var/jenkins_home && \
+    chmod -R 755 /var /var/jenkins_home
+
 # grant user jenkins access to /var/run/docker.sock
 RUN usermod -aG messagebus jenkins
 
 # install plugins
+ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false
 COPY Jenkins.plugins /usr/share/jenkins/ref/plugins.txt
-RUN jenkins-plugin-cli --plugin-file /usr/share/jenkins/ref/plugins.txt
+RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt && \
+    chown -R jenkins:jenkins /var/jenkins_home
 
 USER jenkins

Jenkins/Jenkins.plugins

@@ -1,8 +1,48 @@
-git
+authentication-tokens:latest
-workflow-aggregator
+blueocean:latest
-pipeline-github-lib
+bouncycastle-api:latest
-docker-workflow
+cloudbees-folder:latest
-credentials
+command-launcher:latest
-git-client
+configuration-as-code:latest
-blueocean
+coverage:latest
-coverage
+credentials:latest
+docker-build-step:latest
+docker-commons:latest
+docker-java-api:latest
+docker-plugin:latest
+docker-workflow:latest
+durable-task:latest
+git-client:latest
+git:latest
+instance-identity:latest
+job-dsl:latest
+junit:latest
+matrix-project:latest
+node-iterator-api:latest
+pipeline-build-step:latest
+pipeline-github-lib:latest
+pipeline-groovy-lib:latest
+pipeline-input-step:latest
+pipeline-milestone-step:latest
+pipeline-model-api:latest
+pipeline-model-definition:latest
+pipeline-model-extensions:latest
+pipeline-rest-api:latest
+pipeline-stage-step:latest
+pipeline-stage-tags-metadata:latest
+pipeline-stage-view:latest
+pipeline-utility-steps:latest
+ssh-credentials:latest
+ssh-slaves:latest
+workflow-aggregator:latest
+workflow-basic-steps:latest
+workflow-cps:latest
+workflow-durable-task-step:latest
+workflow-job:latest
+workflow-support:latest
+workflow-step-api:latest
+timestamper:latest
+ws-cleanup:latest
+junit-attachments:latest
+junit-realtime-test-reporter:latest
+

Jenkins/Jenkinsfile

@@ -1,18 +1,27 @@
+def JENKINS_UID = 1000  // UID of jenkins user from Jenkins container
+
 pipeline {
+
     parameters {
         string(name: 'AGENT_CPUS', defaultValue: '2.5', description: 'CPU limit for the build agent')
         string(name: 'AGENT_NETWORK', defaultValue: 'host', description: 'Network to be used for build agent')
         booleanParam(name: 'QUICK_RUN', defaultValue: false, description: 'false: all stages but slow, true: just some stages and fast')
     }
+
     agent {
         dockerfile {
             filename 'Jenkins/jenkins-agent/Dockerfile'
-            args """--user root --network ${params.AGENT_NETWORK}
+            args """--user ${JENKINS_UID} --network ${params.AGENT_NETWORK}
                 --volume /var/run/docker.sock:/var/run/docker.sock
-                    --memory=8g --cpus=${params.AGENT_CPUS}"""
+                --memory=8g --cpus=${params.AGENT_CPUS}
+                --security-opt apparmor=unconfined"""
         }
     }
 
+    options {
+        disableConcurrentBuilds()
+    }
+
     environment {
         GRADLE_USER_HOME = "${env.WORKSPACE}/.gradle-cache"
         DOCKER_HOST = 'unix:///var/run/docker.sock'
@@ -24,7 +33,7 @@ pipeline {
     }
 
     triggers {
-        pollSCM('H/1 * * * *')
+        pollSCM('H/2 * * * *')
     }
 
     stages {
@@ -32,6 +41,7 @@ pipeline {
             steps {
                 sh '''#!/bin/bash +x
                     if command -v docker >/dev/null 2>&1; then
+                      docker info --format '{{.SecurityOptions}}'
                       if docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q 'rootless'; then
                         echo "🟡 Docker daemon is running in ROOTLESS mode"
                       else

Jenkins/Makefile

@@ -1,17 +1,15 @@
 include .env
 export
 
-SOCKET := /var/run/docker.sock
-VOLUME := jenkins_home
-
 CERTBOT_CONF := $(PWD)/.generated/certbot/lib/conf
 CERTBOT_WWW := $(PWD)/.generated/certbot/lib/www
 CERTBOT_LOG := $(PWD)/.generated/certbot/log
 NGINX_LOG := $(PWD)/.generated/certbot/nginx/log
 
-.PHONY: provision \
+.PHONY: provision clean \
-	build run bash init-pw unprotected protected start stop rm purge \
+	jenkins-build jenkins-run jenkins-bash jenkins-init-pw jenkins-unprotected jenkins-protected jenkins-start jenkins-stop jenkins-rm jenkins-purge \
-	nginx-prepare nginx-proxy nginx-start nginx-letsencrypt-init nginx-letsencrypt-timer nginx-restart nginx-stop
+	nginx-prepare nginx-proxy nginx-run nginx-start nginx-letsencrypt-init nginx-letsencrypt-timer nginx-restart nginx-stop \
+	jenkins-security
 
 ## lists all documented targets
 help:
@@ -20,38 +18,51 @@ help:
 		print "    " desc "\n" \
 	}' $(MAKEFILE_LIST)
 
+## uploads to hs.hsadmin.ng/Jenkins/ on the server for testing purposes
+upload:
+	scp -r * .env .gitignore tallyman@$(SERVER_NAME):hs.hsadmin.ng/Jenkins/
+
+
 ## initially, run this once to provision te nginx
-provision: nginx-prepare nginx-letsencrypt-init nginx-letsencrypt-timer nginx-start build start
+provision: nginx-prepare nginx-letsencrypt-init nginx-letsencrypt-timer jenkins-build jenkins-run nginx-restart
+	@echo "now you can start nginx: make nginx-start"
 
 ## removes all generated files
-clean: nginx-stop stop
+clean: nginx-stop jenkins-rm
 	rm -rf .generated/
 
 ## builds the Jenkins image
-build:
+jenkins-build:
 	docker build -t jenkins-docker .
 
-## manually running the Jenkins container
+# initially runs the Jenkins container during provisioning, later use `make jenkins-start`
-run:
+jenkins-run:
+	$(eval DOCKER_SOCKET_MOUNT := $(if $(DOCKER_SOCKET),$(DOCKER_SOCKET):/var/run/docker.sock,/dev/null:/var/run/docker.no-socket))
 	docker run --detach \
 		--dns 8.8.8.8 \
 		--network bridge \
 		--publish 8090:8080 --publish 50000:50000 \
-		--volume $(SOCKET):/var/run/docker.sock \
+		--volume $(DOCKER_SOCKET_MOUNT) \
-		--volume $(VOLUME):/var/jenkins_home \
+		--volume $(JENKINS_VOLUME):/var/jenkins_home \
+		--volume $(PWD)/jenkins.yaml:/var/jenkins_home/jenkins.yaml \
 		--restart unless-stopped \
+		--env-file .env \
 		--name jenkins jenkins-docker
 
 ## manually starts the Jenkins container (again)
-start:
+jenkins-start:
 	docker start jenkins
 
 ## opens a bash within the Jenkins container
-bash:
+jenkins-bash:
 	docker exec -it jenkins bash
 
+## prints the Jenkins log
+jenkins-log:
+	docker logs jenkins 2>&1
+
 ## prints the initial password of a newly setup Jenkins
-init-pw:
+jenkins-init-pw:
 	docker exec -it jenkins sh -c '\
     		while [ ! -f /var/jenkins_home/secrets/initialAdminPassword ]; do \
     			sleep 1; \
@@ -60,50 +71,44 @@ init-pw:
     	'
 
 ## disables security for the Jenkins => allows login to Jenkins without credentials
-unprotected:
+jenkins-unprotected:
 	docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>false</useSecurity>|' /var/jenkins_home/config.xml
 	docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
 
 ## enables security for the Jenkins => Jenkins requires login with credentials
-protected:
+jenkins-protected:
 	docker exec -it jenkins sed -i 's|<useSecurity>true</useSecurity>|<useSecurity>true</useSecurity>|' /var/jenkins_home/config.xml
 	docker exec -it jenkins grep useSecurity /var/jenkins_home/config.xml
 
 ## stops the Jenkins container
-stop:
+jenkins-stop:
-	docker stop jenkins
+	docker stop jenkins || true
 
 ## removes the Jenkins container
-rm: stop
+jenkins-rm: jenkins-stop
-	docker rm jenkins
+	docker rm jenkins || true
 
 ## purges the Jenkins volume (finally deletes the configuration)
-purge: rm
+jenkins-purge: jenkins-rm
-	docker volume rm $(VOLUME)
+	docker volume rm $(JENKINS_VOLUME) || true
 
 # (internal) generates the files for nginx-proxy and certbot
 nginx-prepare:
 	mkdir -p $(CERTBOT_WWW) $(CERTBOT_LOG) $(CERTBOT_CONF)/live/$(SERVER_NAME) $(NGINX_LOG)
 	chmod 755 $(CERTBOT_WWW) $(CERTBOT_LOG) $(CERTBOT_CONF)/live/$(SERVER_NAME) $(NGINX_LOG)
-	sed -e 's/%SERVER_NAME/$(SERVER_NAME)/g' <nginx-proxy/nginx.conf >.generated/nginx.conf
+	sed -e 's/%SERVER_NAME/$(SERVER_NAME)/g' <nginx-proxy/nginx-init.conf >.generated/nginx.conf
 	cp nginx-proxy/options-ssl-nginx.conf $(CERTBOT_CONF)/options-ssl-nginx.conf
 	chmod 644 $(CERTBOT_CONF)/options-ssl-nginx.conf
 	test -f $(CERTBOT_CONF)/ssl-dhparams.pem || curl -o $(CERTBOT_CONF)/ssl-dhparams.pem \
 		https://raw.githubusercontent.com/certbot/certbot/master/certbot/certbot/ssl-dhparams.pem
 	chmod 644 $(CERTBOT_CONF)/ssl-dhparams.pem
-	openssl req -x509 -nodes -newkey rsa:2048 \
-	  -keyout $(CERTBOT_CONF)/live/$(SERVER_NAME)/privkey.pem \
-	  -out /$(CERTBOT_CONF)/live/$(SERVER_NAME)/fullchain.pem \
-	  -subj "/CN=dummy"
 
 ## opens a bash within the Nginx-proxy container
 nginx-bash:
 	docker exec -it nginx bash
 
 # (internal) fetches an initial certificate from letsencrypt
-nginx-letsencrypt-init: nginx-start
+nginx-letsencrypt-init: nginx-run
-	# wait for nginx actually running (could be improved)
-	@sleep 5
 	# delete the previous (dummy) config to avoid file creation with suffix -0001 etc.
 	rm -rf $(CERTBOT_CONF)/etc/letsencrypt/live/$(SERVER_NAME) \
        $(CERTBOT_CONF)/etc/letsencrypt/archive/$(SERVER_NAME) \
@@ -114,13 +119,12 @@ nginx-letsencrypt-init: nginx-start
 	  -v $(CERTBOT_WWW):/var/www/certbot \
 	  -v $(CERTBOT_LOG):/var/log/letsencrypt \
 	  certbot/certbot \
-	  certonly --webroot --webroot-path /var/www/certbot \
+	  certonly --webroot --webroot-path /var/www/certbot --cert-name $(SERVER_NAME) \
-	  --email $(EMAIL) --cert-name $(SERVER_NAME) \
 	  -d $(SERVER_NAME) --rsa-key-size 4096 \
-	  --agree-tos --force-renewal
+	  --non-interactive --agree-tos --force-renewal $(CERTBOT_ENV)
-	# restart nginx
+	# from now on, start nginx including https
+	sed -e 's/%SERVER_NAME/$(SERVER_NAME)/g' <nginx-proxy/nginx.conf >.generated/nginx.conf
 	docker stop nginx || true
-	docker start nginx
 
 ## opens a shell in the letsencrypt certbot
 nginx-letsencrypt-sh:
@@ -147,8 +151,8 @@ nginx-letsencrypt-renew:
 	  -v $(CERTBOT_LOG):/var/log/letsencrypt \
 	  certbot/certbot renew -q
 
-## starts the nginx proxy server
+## initially runs the nginx proxy server
-nginx-start: nginx-stop
+nginx-run: nginx-stop
 	docker run -d --name nginx \
 	  --publish 8080:80 \
 	  --publish 8443:443 \
@@ -157,8 +161,16 @@ nginx-start: nginx-stop
 	  -v $(CERTBOT_WWW):/var/www/certbot \
 	  -v $(NGINX_LOG):/var/log/nginx \
 	  -v $(PWD)/.generated/nginx.conf:/etc/nginx/nginx.conf \
+	  --health-cmd="curl -kfs https://localhost:8443/ || exit 1" \
+	  --health-interval=5s \
+	  --health-timeout=3s \
+	  --health-retries=3 \
 	  nginx
 
+## starts the nginx proxy server again
+nginx-start:
+	docker start nginx
+
 ## restarts the nginx proxy server
 nginx-restart: nginx-stop nginx-start
 
@@ -167,3 +179,15 @@ nginx-stop:
 	docker stop nginx || true
 	docker rm nginx || true
 
+## remove the nginx container
+nginx-rm: nginx-stop
+	docker rm nginx || true
+
+## check security status
+jenkins-security:
+	@curl --insecure -s -o /dev/null -w "%{http_code}\n" https://localhost:8443/script
+
+## fix access rights in workspaces
+jenkins-fix:
+	@docker run --rm -it -v $(JENKINS_VOLUME):/mnt alpine chown 1000:1000 -R /mnt/workspace
+

Jenkins/README.md

@@ -5,12 +5,24 @@ The scripts work in a Hostsharing Managed Docker environment.
 Requires a .env file like this in the current directory: 
 
 ```
+DOCKER_SOCKET=/var/run/docker.sock
+DOCKER_HOST=unix:///var/run/docker.soc
 SERVER_NAME=jenkins.example.org
-EMAIL=contact@example.org
+JENKINS_VOLUME=jenkins_home
+JENKINS_ADMIN_PASSWORD=password-for-initial-user-admin
+GIT_USERNAME=git-username
+GIT_PASSWORD=git-password
+CERTBOT_ENV=--staging # leave empty for real certificates or --staging for test certificates
 ```
 
 Then run `make provision` to initialize everything.
 
-Run `make help` for more information.
+To completely start over again, run `make jenkins-purge clean provision`.
+This will also remove all Jenkins configurations!
 
-WARNING: Provisioning does not really work yet, needs some manual restarts.
+Once everything works, you can remove `--staging` from `.env`
+and run `make clean provision`.
+Now, a *letsencrypt* is asked to issue a real certificate.
+Beware, this is only possible 5 times per 24h.
+
+Run `make help` for more information.

Jenkins/jenkins-agent/Dockerfile

@@ -1,4 +1,10 @@
 FROM eclipse-temurin:21-jdk
+
+# create mount point for jenkins_home
+RUN mkdir -p /var/jenkins_home && \
+    chmod 755 /var/jenkins_home
+
+# install required packages
 RUN apt-get update && \
     apt-get install -y \
         postgresql-client \
@@ -7,3 +13,6 @@ RUN apt-get update && \
         pandoc && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+
+# continue with the same uid as the user 'jenkins' from the Jenkins Dockerfile
+USER 1000

Jenkins/jenkins.yaml

@@ -0,0 +1,118 @@
+jenkins:
+  systemMessage: "Jenkins configuration via Jenkins Configuration as Code"
+  authorizationStrategy:
+    loggedInUsersCanDoAnything:
+      allowAnonymousRead: true
+  clouds:
+    - docker:
+        name: "docker"
+        dockerApi:
+          dockerHost:
+            uri: "${DOCKER_HOST}"
+          connectTimeout: 60
+          readTimeout: 60
+        containerCap: 10
+  remotingSecurity:
+    enabled: true
+  securityRealm:
+    local:
+      allowsSignup: false
+      enableCaptcha: false
+      users:
+        - id: "admin"
+          name: "admin"
+          password: "${JENKINS_ADMIN_PASSWORD}"
+          properties:
+            - "consoleUrlProvider"
+            - "favorite"
+            - "myView"
+            - preferredProvider:
+                providerId: "default"
+            - theme:
+                theme: "noOp"
+            - "timezone"
+            - "experimentalFlags"
+            - mailer:
+                emailAddress: "michael.hoennig@hostsharing.net"
+            - "apiToken"
+
+credentials:
+  system:
+    domainCredentials:
+      - credentials:
+          # Username/password credential
+          - usernamePassword:
+              scope: GLOBAL
+              id: 'hsadmin-NG-git'
+              username: "${GIT_USERNAME}"
+              password: "${GIT_PASSWORD}"
+              description: 'git access'
+
+jobs:
+    - script: >
+          multibranchPipelineJob('hsadmin-NG Java backend') {
+            branchSources {
+              git {
+                id('hsadmin-NG')
+                remote('https://dev.hostsharing.net/hostsharing/hs.hsadmin.ng')
+                credentialsId('hsadmin-NG-git')
+              }
+            }
+            factory {
+              workflowBranchProjectFactory {
+                scriptPath('Jenkins/Jenkinsfile')
+              }
+            }
+            triggers {
+              periodicFolderTrigger {
+                interval('2m')
+              }
+            }
+          }
+
+security:
+  apiToken:
+    creationOfLegacyTokenEnabled: false
+    tokenGenerationOnCreationEnabled: false
+    usageStatisticsEnabled: true
+  cps:
+    hideSandbox: false
+  gitHooks:
+    allowedOnAgents: false
+    allowedOnController: false
+  gitHostKeyVerificationConfiguration:
+    sshHostKeyVerificationStrategy: "knownHostsFileVerificationStrategy"
+  globalJobDslSecurityConfiguration:
+    useScriptSecurity: true
+  scriptApproval:
+    forceSandbox: true
+  location:
+    adminAddress: "michael.hoennig@hostsharing.net"
+    url: "https://vm2176.hostsharing.net/"
+  mailer:
+    charset: "UTF-8"
+    useSsl: false
+    useTls: false
+  pollSCM:
+    pollingThreadCount: 10
+  scmGit:
+    addGitTagAction: false
+    allowSecondFetch: false
+    createAccountBasedOnEmail: false
+    disableGitToolChooser: false
+    hideCredentials: false
+    showEntireCommitSummaryInChanges: false
+    useExistingAccountWithSameEmail: false
+  timestamper:
+    allPipelines: false
+    elapsedTimeFormat: "'<b>'HH:mm:ss.S'</b> '"
+    systemTimeFormat: "'<b>'HH:mm:ss'</b> '"
+
+tool:
+  git:
+    installations:
+      - home: "git"
+        name: "Default"
+  mavenGlobalConfig:
+    globalSettingsProvider: "standard"
+    settingsProvider: "standard"

Jenkins/nginx-proxy/nginx-init.conf

@@ -0,0 +1,19 @@
+events {}
+
+http {
+    server {
+        listen 80;
+        server_name %SERVER_NAME;
+
+        # directly answer initial certbot request
+        location /.well-known/acme-challenge/ {
+            root /var/www/certbot;
+        }
+
+        # forward all other HTTP-requests to HTTPS
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+}
+