migrate hosting-assets into external db (#158)

Co-authored-by: Michael Hoennig <michael@hoennig.de>
Reviewed-on: https://dev.hostsharing.net/hostsharing/hs.hsadmin.ng/pulls/158
Reviewed-by: Marc Sandlus <marc.sandlus@hostsharing.net>

.aliases

@@ -113,9 +113,9 @@ function _gwTest1() {
     echo "DONE gw $@"
 }
 function _gwTest() {
-  . .aliases;
-  . .tc-environment;
-  rm /tmp/gwTest.tmp
+  . .aliases
+  . .tc-environment
+  rm -f /tmp/gwTest.tmp
   if [ "$1" == "--all" ]; then
     shift # to remove the --all from $@
     # delierately in separate gradlew-calls to avoid Testcontains-PostgreSQL problem spillover

.run/ImportHostingAssets into local.run.xml

@@ -7,6 +7,7 @@
           <entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="postgres" />
           <entry key="HSADMINNG_POSTGRES_JDBC_URL" value="jdbc:postgresql://localhost:5432/postgres" />
           <entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
+          <entry key="HSADMINNG_MIGRATION_DATA_PATH" value="migration" />
         </map>
       </option>
       <option name="executionName" />
@@ -34,4 +35,4 @@
     <RunAsTest>true</RunAsTest>
     <method v="2" />
   </configuration>
-</component>
+</component>

.run/ImportHostingAssets.run.xml

@@ -3,7 +3,6 @@
     <ExternalSystemSettings>
       <option name="env">
         <map>
-          <entry key="HSADMINNG_MIGRATION_DATA_PATH" value="migration" />
           <entry key="HSADMINNG_POSTGRES_ADMIN_USERNAME" value="admin" />
           <entry key="HSADMINNG_POSTGRES_RESTRICTED_USERNAME" value="restricted" />
           <entry key="HSADMINNG_SUPERUSER" value="import-superuser@hostsharing.net" />

.tc-environment

@@ -1,7 +1,7 @@
-unset HSADMINNG_POSTGRES_JDBC_URL # dynamically set, different for normal tests and imports
-export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
-export HSADMINNG_POSTGRES_ADMIN_PASSWORD=
+source .unset-environment
+
 export HSADMINNG_POSTGRES_RESTRICTED_USERNAME=restricted
-export HSADMINNG_SUPERUSER=superuser-alex@hostsharing.net
-export HSADMINNG_MIGRATION_DATA_PATH=migration
+export HSADMINNG_POSTGRES_ADMIN_USERNAME=admin
+export HSADMINNG_SUPERUSER=import-superuser@hostsharing.net
+
 export LANG=en_US.UTF-8

.unset-environment

@@ -4,4 +4,5 @@ unset HSADMINNG_POSTGRES_ADMIN_PASSWORD
 unset HSADMINNG_POSTGRES_RESTRICTED_USERNAME
 unset HSADMINNG_SUPERUSER
 unset HSADMINNG_MIGRATION_DATA_PATH
+unset HSADMINNG_OFFICE_DATA_SQL_FILE
 

src/main/resources/db/changelog/1-rbac/1055-rbac-views.sql

@@ -22,13 +22,12 @@ select (objectTable || '#' || objectIdName || ':' || roleType) as roleIdName, *
 --//
 
 -- ============================================================================
---changeset michael.hoennig:rbac-views-ROLE-RESTRICTED-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-views-ROLE-RESTRICTED-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 /*
     Creates a view to the role table with row-level limitation
     based on the grants of the current user or assumed roles.
  */
-drop view if exists rbac.role_rv;
 create or replace view rbac.role_rv as
 select *
        -- @formatter:off
@@ -106,7 +105,7 @@ create or replace view rbac.grant_ev as
 
 
 -- ============================================================================
---changeset michael.hoennig:rbac-views-GRANT-RESTRICTED-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-views-GRANT-RESTRICTED-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 /*
     Creates a view to the grants table with row-level limitation
@@ -222,13 +221,12 @@ select distinct *
 
 
 -- ============================================================================
---changeset michael.hoennig:rbac-views-USER-RESTRICTED-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-views-USER-RESTRICTED-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 /*
     Creates a view to the users table with row-level limitation
     based on the grants of the current user or assumed roles.
  */
-drop view if exists rbac.subject_rv;
 create or replace view rbac.subject_rv as
     select distinct *
         -- @formatter:off
@@ -316,14 +314,13 @@ execute function rbac.delete_subject_tf();
 --/
 
 -- ============================================================================
---changeset michael.hoennig:rbac-views-OWN-GRANTED-PERMISSIONS-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-views-OWN-GRANTED-PERMISSIONS-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 /*
     Creates a view to all permissions granted to the current user or
     based on the grants of the current user or assumed roles.
  */
 -- @formatter:off
-drop view if exists rbac.own_granted_permissions_rv;
 create or replace view rbac.own_granted_permissions_rv as
 select r.uuid as roleuuid, p.uuid as permissionUuid,
        (r.objecttable || ':' || r.objectidname || ':' ||  r.roletype) as roleName, p.op,

src/main/resources/db/changelog/1-rbac/1058-rbac-generators.sql

@@ -111,7 +111,7 @@ end; $$;
 
 
 -- ============================================================================
---changeset michael.hoennig:rbac-generators-IDENTITY-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-generators-IDENTITY-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 
 create or replace procedure rbac.generateRbacIdentityViewFromQuery(targetTable text, sqlQuery text)
@@ -171,7 +171,7 @@ end; $$;
 
 
 -- ============================================================================
---changeset michael.hoennig:rbac-generators-RESTRICTED-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-generators-RESTRICTED-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 
 create or replace procedure rbac.generateRbacRestrictedView(targetTable text, orderBy text, columnUpdates text = null, columnNames text = '*')

src/main/resources/db/changelog/1-rbac/1080-rbac-global.sql

@@ -1,7 +1,7 @@
 --liquibase formatted sql
 
 -- ============================================================================
---changeset michael.hoennig:rbac-global-OBJECT endDelimiter:--//
+--changeset michael.hoennig:rbac-global-OBJECT runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 /*
     The purpose of this table is provide root business objects
@@ -11,12 +11,12 @@
     In production databases, there is only a single row in this table,
     in test stages, there can be one row for each test data realm.
  */
-create table rbac.global
+create table if not exists rbac.global
 (
     uuid uuid primary key references rbac.object (uuid) on delete cascade,
     name varchar(63) unique
 );
-create unique index Global_Singleton on rbac.global ((0));
+create unique index if not exists Global_Singleton on rbac.global ((0));
 
 grant select on rbac.global to ${HSADMINNG_POSTGRES_RESTRICTED_USERNAME};
 --//
@@ -75,13 +75,12 @@ $$;
 
 
 -- ============================================================================
---changeset michael.hoennig:rbac-global-IDENTITY-VIEW endDelimiter:--//
+--changeset michael.hoennig:rbac-global-IDENTITY-VIEW runOnChange:true validCheckSum:ANY endDelimiter:--//
 -- ----------------------------------------------------------------------------
 
 /*
     Creates a view to the rbac.global object table which maps the identifying name to the objectUuid.
  */
-drop view if exists rbac.global_iv;
 create or replace view rbac.global_iv as
 select target.uuid, target.name as idName
     from rbac.global as target;

src/test/java/net/hostsharing/hsadminng/hs/migration/CsvDataImport.java

@@ -16,6 +16,9 @@ import org.junit.jupiter.api.extension.TestWatcher;
 import org.opentest4j.AssertionFailedError;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.AbstractResource;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.FileSystemResource;
 import org.springframework.test.context.bean.override.mockito.MockitoBean;
 import org.springframework.core.io.Resource;
 import org.springframework.transaction.support.TransactionTemplate;
@@ -26,6 +29,7 @@ import jakarta.servlet.http.HttpServletRequest;
 import jakarta.validation.ValidationException;
 import jakarta.validation.constraints.NotNull;
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
@@ -118,10 +122,16 @@ public class CsvDataImport extends ContextBasedTest {
         return stream(lines.getFirst()).map(String::trim).toArray(String[]::new);
     }
 
+    public static @NotNull AbstractResource resourceOf(final String sqlFile) {
+        return new File(sqlFile).exists()
+                ? new FileSystemResource(sqlFile)
+                : new ClassPathResource(sqlFile);
+    }
+
     protected Reader resourceReader(@NotNull final String resourcePath) {
         try {
-            return new InputStreamReader(requireNonNull(getClass().getClassLoader().getResourceAsStream(resourcePath)));
-        } catch (Exception exc) {
+            return new InputStreamReader(requireNonNull(resourceOf(resourcePath).getInputStream()));
+        } catch (final Exception exc) {
             throw new AssertionFailedError("cannot open '" + resourcePath + "'");
         }
     }

src/test/java/net/hostsharing/hsadminng/hs/migration/ImportHostingAssets.java

@@ -12,6 +12,7 @@ import net.hostsharing.hsadminng.hs.booking.item.HsBookingItem;
 import net.hostsharing.hsadminng.hs.booking.item.HsBookingItemRealEntity;
 import net.hostsharing.hsadminng.hs.booking.item.HsBookingItemType;
 import net.hostsharing.hsadminng.hs.booking.item.validators.HsBookingItemEntityValidatorRegistry;
+import net.hostsharing.hsadminng.hs.booking.project.HsBookingProject;
 import net.hostsharing.hsadminng.hs.booking.project.HsBookingProjectRealEntity;
 import net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetRealEntity;
 import net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType;
@@ -29,14 +30,18 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
 import org.springframework.context.annotation.Import;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.orm.jpa.EntityManagerFactoryInfo;
 import org.springframework.test.annotation.Commit;
 import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ActiveProfiles;
-import org.springframework.test.context.jdbc.Sql;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Reader;
 import java.net.IDN;
 import java.util.ArrayList;
@@ -52,6 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Arrays.stream;
 import static java.util.Map.entry;
 import static java.util.Map.ofEntries;
@@ -80,7 +86,7 @@ import static net.hostsharing.hsadminng.hs.hosting.asset.HsHostingAssetType.UNIX
 import static net.hostsharing.hsadminng.mapper.PostgresDateRange.toPostgresDateRange;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assumptions.assumeThat;
-import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TEST_CLASS;
+import static org.springframework.util.FileCopyUtils.copyToByteArray;
 
 @Tag("importHostingAssets")
 @DataJpaTest(properties = {
@@ -95,7 +101,6 @@ import static org.springframework.test.context.jdbc.Sql.ExecutionPhase.BEFORE_TE
 @ActiveProfiles({ "without-test-data", "liquibase-migration", "hosting-asset-import" })
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 @ExtendWith(OrderedDependedTestsExtension.class)
-@Sql(value = "/db/released-only-office-schema-with-import-test-data.sql", executionPhase = BEFORE_TEST_CLASS) // release-schema
 public class ImportHostingAssets extends CsvDataImport {
 
     private static final Set<String> NOBODY_SUBSTITUTES = Set.of("nomail", "bounce");
@@ -133,9 +138,14 @@ public class ImportHostingAssets extends CsvDataImport {
     @Autowired
     LiquibaseMigration liquibase;
 
+    @Value("${HSADMINNG_OFFICE_DATA_SQL_FILE:/db/released-only-office-schema-with-import-test-data.sql}")
+    String officeSchemaAndDataSqlFile;
+
     @Test
     @Order(11000)
+    @SneakyThrows
     void liquibaseMigrationForBookingAndHosting() {
+        executeSqlScript(officeSchemaAndDataSqlFile);
         liquibase.assertReferenceStatusAfterRestore(286, "hs-booking-SCHEMA");
         makeSureThatTheImportAdminUserExists();
         liquibase.runWithContexts("migration", "without-test-data");
@@ -146,8 +156,8 @@ public class ImportHostingAssets extends CsvDataImport {
     @Order(11010)
     void createBookingProjects() {
 
-        record PartnerLegacyIdMapping(UUID uuid, Integer bp_id){}
-        record DebitorRecord(UUID uuid, Integer version, String defaultPrefix){}
+        record PartnerLegacyIdMapping(UUID uuid, Integer bp_id) {}
+        record DebitorRecord(UUID uuid, Integer version, String defaultPrefix) {}
 
         final var partnerLegacyIdMappings = em.createNativeQuery(
                 """
@@ -161,16 +171,18 @@ public class ImportHostingAssets extends CsvDataImport {
         //noinspection unchecked
         final var debitorUuidToLegacyBpIdMap = ((List<PartnerLegacyIdMapping>) partnerLegacyIdMappings).stream()
                 .collect(toMap(row -> row.uuid, row -> row.bp_id));
-        final var debitors = em.createNativeQuery("SELECT debitor.uuid, debitor.version, debitor.defaultPrefix FROM hs_office.debitor debitor", DebitorRecord.class).getResultList();
+        final var debitors = em.createNativeQuery(
+                "select debitor.uuid, debitor.version, debitor.defaultPrefix from hs_office.debitor debitor",
+                DebitorRecord.class).getResultList();
         //noinspection unchecked
-        ((List<DebitorRecord>)debitors).forEach(debitor -> {
-                    bookingProjects.put(
-                            debitorUuidToLegacyBpIdMap.get(debitor.uuid), HsBookingProjectRealEntity.builder()
-                                    .version(debitor.version)
-                                    .caption(debitor.defaultPrefix + " default project")
-                                    .debitor(em.find(HsBookingDebitorEntity.class, debitor.uuid))
-                                    .build());
-                });
+        ((List<DebitorRecord>) debitors).forEach(debitor -> {
+            bookingProjects.put(
+                    debitorUuidToLegacyBpIdMap.get(debitor.uuid), HsBookingProjectRealEntity.builder()
+                            .version(debitor.version)
+                            .caption(debitor.defaultPrefix + " default project")
+                            .debitor(em.find(HsBookingDebitorEntity.class, debitor.uuid))
+                            .build());
+        });
     }
 
     @Test
@@ -1231,9 +1243,7 @@ public class ImportHostingAssets extends CsvDataImport {
                     bookingItems.put(packet_id, bookingItem);
                     final var haType = determineHaType(basepacket_code);
 
-                    logError(() -> assertThat(!free || haType == MANAGED_WEBSPACE || bookingItem.getRelatedProject()
-                            .getDebitor()
-                            .getDefaultPrefix()
+                    logError(() -> assertThat(!free || haType == MANAGED_WEBSPACE || defaultPrefix(bookingItem)
                             .equals("hsh"))
                             .as("packet.free only supported for Hostsharing-Assets and ManagedWebspace in customer-ManagedServer, but is set for "
                                     + packet_name)
@@ -1288,6 +1298,13 @@ public class ImportHostingAssets extends CsvDataImport {
                 });
     }
 
+    private String defaultPrefix(final HsBookingItem bookingItem) {
+        return ofNullable(bookingItem.getProject())
+                .map(HsBookingProject::getDebitor)
+                .map(HsBookingDebitorEntity::getDefaultPrefix)
+                .orElse("<no default prefix for BI: " + bookingItem.getCaption() + ">");
+    }
+
     private void importPacketComponents(final String[] header, final List<String[]> records) {
         final var columns = new Columns(header);
         records.stream()
@@ -1940,4 +1957,17 @@ public class ImportHostingAssets extends CsvDataImport {
                 .map(row -> row.stream().map(Object::toString).collect(joining(", ")))
                 .collect(joining("\n"));
     }
+
+    @SneakyThrows
+    private void executeSqlScript(final String sqlFile) {
+        jpaAttempt.transacted(() -> {
+            try (InputStream resourceStream = resourceOf(sqlFile).getInputStream()) {
+                final var sqlScript = new String(copyToByteArray(resourceStream), UTF_8);
+                final var emf = (EntityManagerFactoryInfo) em.getEntityManagerFactory();
+                new JdbcTemplate(emf.getDataSource()).execute(sqlScript);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }).assertSuccessful();
+    }
 }