Added logging; Added Migrations

pom.xml

@@ -24,6 +24,12 @@
             <artifactId>postgresql</artifactId>
             <version>42.2.9</version>
         </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
+
     </dependencies>
 
 </project>

src/main/java/Main.java

@@ -1,17 +1,21 @@
 import data.source.PersonSource;
 import data.target.CharacterTarget;
 import etl.*;
+import migration.ThesisMigration;
+import org.apache.log4j.Logger;
 import utils.*;
 
 import java.util.*;
 
 public class Main
 {
+    private static Logger log = Logger.getLogger(Main.class.getName());
+
     public static void main(final String[] args)
     {
-        var p = testExtract();
-        var pt = testTransform(p);
-        testLoad(pt);
+        log.info("------------- Starting Migration -------------");
+        new ThesisMigration().executeMigrations();
+        log.info("------------- Migration Finished -------------");
     }
 
     private static List<PersonSource> testExtract()

src/main/java/data/source/PlayerAbilitiesSource.java

@@ -0,0 +1,32 @@
+package data.source;
+
+import data.SourceDataset;
+
+public class PlayerAbilitiesSource implements SourceDataset
+{
+    private int playerId;
+    private int abilityId;
+
+    public PlayerAbilitiesSource(int playerId, int abilityId)
+    {
+        this.playerId = playerId;
+        this.abilityId = abilityId;
+    }
+
+    public int getPlayerId()
+    {
+        return playerId;
+    }
+
+    public int getAbilityId()
+    {
+        return abilityId;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("PlayerAbility [ %d, %d ]", this.playerId, this.abilityId);
+    }
+
+}

src/main/java/etl/Extractor.java

@@ -1,6 +1,7 @@
 package etl;
 
 import data.SourceDataset;
+import org.apache.log4j.Logger;
 import utils.DataStorer;
 import utils.StatementPreparerExtractor;
 
@@ -10,6 +11,8 @@ import java.util.List;
 
 public class Extractor<T extends SourceDataset>
 {
+    private static Logger log = Logger.getLogger(Extractor.class.getName());
+
     private Connection sourceDatabase;
     private DataStorer<T> dataStorer;
     private StatementPreparerExtractor statementPreparer;
@@ -31,7 +34,7 @@ public class Extractor<T extends SourceDataset>
             var preparedStatement = this.sourceDatabase.prepareStatement(sql);
             this.statementPreparer.doPrepare(preparedStatement);
             var resultSet = preparedStatement.executeQuery();
-            this.sourceDatabase.close();
+            log.info(String.format("--- Data extracted with '%s' ---", this.sql));
             return this.dataStorer.doStore(resultSet);
         }
         catch (SQLException e)

src/main/java/etl/Loader.java

@@ -2,6 +2,7 @@ package etl;
 
 import data.SourceDataset;
 import data.TargetDataset;
+import org.apache.log4j.Logger;
 import utils.StatementPreparerLoader;
 
 import java.sql.Connection;
@@ -10,6 +11,8 @@ import java.util.List;
 
 public class Loader<T extends TargetDataset>
 {
+    private static Logger log = Logger.getLogger(Loader.class.getName());
+
     private Connection targetDatabase;
     private StatementPreparerLoader<T> statementPreparerLoader;
     private List<T> transformedData;
@@ -30,12 +33,12 @@ public class Loader<T extends TargetDataset>
         {
             for (T transformedDatum : this.transformedData)
             {
+                log.info(String.format("Load data into target: %s", transformedDatum));
                 var preparedStatement = this.targetDatabase.prepareStatement(this.sql);
                 this.statementPreparerLoader.doPrepare(preparedStatement, transformedDatum);
                 preparedStatement.executeUpdate();
             }
-
-            this.targetDatabase.close();
+            log.info(String.format("--- Data loaded into target with '%s' ---", this.sql));
         }
         catch (SQLException e)
         {

src/main/java/etl/Transformer.java

@@ -2,6 +2,7 @@ package etl;
 
 import data.SourceDataset;
 import data.TargetDataset;
+import org.apache.log4j.Logger;
 import utils.DataTransformer;
 
 import java.util.ArrayList;
@@ -9,6 +10,7 @@ import java.util.List;
 
 public class Transformer<T extends SourceDataset, E extends TargetDataset>
 {
+    private static Logger log = Logger.getLogger(Transformer.class.getName());
 
     private DataTransformer<T, E> transformer;
     private List<T> extractedData;
@@ -24,8 +26,10 @@ public class Transformer<T extends SourceDataset, E extends TargetDataset>
         var transformed = new ArrayList<E>();
         for (T dataset : this.extractedData)
         {
+            log.info(String.format("Transform data: %s", dataset));
             transformed.add(this.transformer.transform(dataset));
         }
+        log.info("--- Data transformed ---");
         return transformed;
     }
 }

src/main/java/migration/AbilityMigration.java

@@ -0,0 +1,76 @@
+package migration;
+
+import data.source.AbilitiesSource;
+import data.source.PersonSource;
+import data.target.AbilityTarget;
+import data.target.CharacterTarget;
+import etl.Extractor;
+import etl.Loader;
+import etl.Transformer;
+import utils.*;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+public class AbilityMigration extends ETL<AbilitiesSource, AbilityTarget>
+{
+    private int abilityId;
+
+    public AbilityMigration(Connection source, Connection target)
+    {
+        super(source, target);
+        this.abilityId = 0;
+    }
+
+    @Override
+    protected List<AbilitiesSource> extract()
+    {
+        DataStorer<AbilitiesSource> dataStorer = (resultSet) -> {
+            var extractedData = new ArrayList<AbilitiesSource>();
+            while (resultSet.next())
+            {
+                extractedData.add(new AbilitiesSource(
+                        resultSet.getString("name"),
+                        resultSet.getString("description"),
+                        resultSet.getInt("level")
+                ));
+            }
+            return extractedData;
+        };
+
+        var sql = "select * from abilities;";
+        StatementPreparerExtractor statementPreparer = (preparedStatement) -> {};
+
+        return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract();
+    }
+
+    @Override
+    protected List<AbilityTarget> transform(List<AbilitiesSource> extractedData)
+    {
+        DataTransformer<AbilitiesSource, AbilityTarget> transformer =
+                (dataset) -> new AbilityTarget(
+                        abilityId++,
+                        dataset.getName(),
+                        dataset.getDescription(),
+                        dataset.getLevel()
+                );
+
+        return new Transformer<>(transformer, extractedData).doTransform();
+    }
+
+    @Override
+    protected void load(List<AbilityTarget> transformedData)
+    {
+        StatementPreparerLoader<AbilityTarget> statementPreparerLoader =  (preparedStatement, data) -> {
+            preparedStatement.setInt(1, data.getAbilityId());
+            preparedStatement.setString(2, data.getAbilityName());
+            preparedStatement.setString(3, data.getAbilityDescription());
+            preparedStatement.setInt(4, data.getAbilityLevel());
+        };
+
+        var sql = "insert into ability values (?, ?, ?, ?)";
+
+        new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad();
+    }
+}

src/main/java/migration/CharacterMigration.java

@@ -0,0 +1,74 @@
+package migration;
+
+import data.source.AbilitiesSource;
+import data.source.PersonSource;
+import data.target.AbilityTarget;
+import data.target.CharacterTarget;
+import etl.Extractor;
+import etl.Loader;
+import etl.Transformer;
+import utils.DataStorer;
+import utils.DataTransformer;
+import utils.StatementPreparerExtractor;
+import utils.StatementPreparerLoader;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CharacterMigration extends ETL<PersonSource, CharacterTarget>
+{
+    public CharacterMigration(Connection source, Connection target)
+    {
+        super(source, target);
+    }
+
+    @Override
+    protected List<PersonSource> extract()
+    {
+        DataStorer<PersonSource> dataStorer = (resultSet) -> {
+            var extractedData = new ArrayList<PersonSource>();
+            while (resultSet.next())
+            {
+                extractedData.add(new PersonSource(
+                        resultSet.getInt("personId"),
+                        resultSet.getString("name"),
+                        resultSet.getBoolean("mortal")
+                ));
+            }
+            return extractedData;
+        };
+
+        var sql = "select * from person;";
+        StatementPreparerExtractor statementPreparer = (preparedStatement) -> {};
+
+        return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract();
+    }
+
+    @Override
+    protected List<CharacterTarget> transform(List<PersonSource> extractedData)
+    {
+        DataTransformer<PersonSource, CharacterTarget> transformer =
+                (dataset) -> new CharacterTarget(
+                        dataset.getPersonId(),
+                        dataset.getName(),
+                        dataset.isMortal()
+                );
+
+        return new Transformer<>(transformer, extractedData).doTransform();
+    }
+
+    @Override
+    protected void load(List<CharacterTarget> transformedData)
+    {
+        StatementPreparerLoader<CharacterTarget> statementPreparerLoader =  (preparedStatement, data) -> {
+            preparedStatement.setInt(1, data.getPersonId());
+            preparedStatement.setString(2, data.getName());
+            preparedStatement.setBoolean(3, data.isMortal());
+        };
+
+        var sql = "insert into character values (?, ?, ?)";
+
+        new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad();
+    }
+}

src/main/java/migration/ETL.java

@@ -0,0 +1,35 @@
+package migration;
+
+import data.SourceDataset;
+import data.TargetDataset;
+import data.source.PersonSource;
+import org.apache.log4j.Logger;
+
+import java.sql.Connection;
+import java.util.List;
+
+public abstract class ETL<T extends SourceDataset, E extends TargetDataset>
+{
+    protected static Logger log = Logger.getLogger(ETL.class.getName());
+
+    protected Connection source;
+    protected Connection target;
+
+    public ETL(Connection source, Connection target)
+    {
+        this.source = source;
+        this.target = target;
+    }
+
+    public void migrate()
+    {
+        log.info(String.format("--- Migration: %s ---", this.getClass().getName()));
+        var extractedData = this.extract();
+        var transformedData = this.transform(extractedData);
+        this.load(transformedData);
+    }
+
+    protected abstract List<T> extract();
+    protected abstract List<E> transform(List<T> extractedData);
+    protected abstract void load(List<E> transformedData);
+}

src/main/java/migration/GameobjectMigration.java

@@ -0,0 +1,33 @@
+package migration;
+
+import data.source.GameobjectSource;
+import data.target.GameobjectTarget;
+
+import java.sql.Connection;
+import java.util.List;
+
+public class GameobjectMigration extends ETL<GameobjectSource, GameobjectTarget>
+{
+    public GameobjectMigration(Connection source, Connection target)
+    {
+        super(source, target);
+    }
+
+    @Override
+    protected List<GameobjectSource> extract()
+    {
+        return null;
+    }
+
+    @Override
+    protected List<GameobjectTarget> transform(List<GameobjectSource> extractedData)
+    {
+        return null;
+    }
+
+    @Override
+    protected void load(List<GameobjectTarget> transformedData)
+    {
+
+    }
+}

src/main/java/migration/PlayerAbilitiesMigration.java

@@ -0,0 +1,72 @@
+package migration;
+
+import data.SourceDataset;
+import data.source.AbilitiesSource;
+import data.source.PlayerAbilitiesSource;
+import data.target.AbilityTarget;
+import data.target.PlayerAbilitiesTarget;
+import etl.Extractor;
+import etl.Loader;
+import etl.Transformer;
+import utils.DataStorer;
+import utils.DataTransformer;
+import utils.StatementPreparerExtractor;
+import utils.StatementPreparerLoader;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+public class PlayerAbilitiesMigration extends ETL<PlayerAbilitiesSource, PlayerAbilitiesTarget>
+{
+    public PlayerAbilitiesMigration(Connection source, Connection target)
+    {
+        super(source, target);
+    }
+
+    @Override
+    protected List<PlayerAbilitiesSource> extract()
+    {
+        DataStorer<PlayerAbilitiesSource> dataStorer = (resultSet) -> {
+            var extractedData = new ArrayList<PlayerAbilitiesSource>();
+            while (resultSet.next())
+            {
+                extractedData.add(new PlayerAbilitiesSource(
+                        resultSet.getInt("playerId"),
+                        resultSet.getInt("abilityId")
+                ));
+            }
+            return extractedData;
+        };
+
+        var sql = "select * from player p join ability a on p.playerId = ?;";
+        StatementPreparerExtractor statementPreparer = (preparedStatement) -> {
+            preparedStatement.setInt(1, 0);
+        };
+
+        return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract();
+    }
+
+    @Override
+    protected List<PlayerAbilitiesTarget> transform(List<PlayerAbilitiesSource> extractedData)
+    {
+        DataTransformer<PlayerAbilitiesSource, PlayerAbilitiesTarget> transformer =
+                (dataset) -> new PlayerAbilitiesTarget(dataset.getPlayerId(), dataset.getAbilityId());
+
+        return new Transformer<>(transformer, extractedData).doTransform();
+    }
+
+    @Override
+    protected void load(List<PlayerAbilitiesTarget> transformedData)
+    {
+
+        StatementPreparerLoader<PlayerAbilitiesTarget> statementPreparerLoader =  (preparedStatement, data) -> {
+            preparedStatement.setInt(1, data.getPlayerId());
+            preparedStatement.setInt(2, data.getAbilityId());
+        };
+
+        var sql = "insert into playerAbilities values (?, ?)";
+
+        new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad();
+    }
+}

src/main/java/migration/PlayerMigration.java

@@ -0,0 +1,57 @@
+package migration;
+
+import data.SourceDataset;
+import data.target.PlayerTarget;
+import etl.Loader;
+import utils.ConnectionHelper;
+import utils.DatabaseInformation;
+import utils.DatabaseType;
+import utils.StatementPreparerLoader;
+
+import java.sql.Connection;
+import java.util.List;
+
+public class PlayerMigration extends ETL<SourceDataset, PlayerTarget>
+{
+    public PlayerMigration(Connection source, Connection target)
+    {
+        super(source, target);
+    }
+
+    @Override
+    public void migrate()
+    {
+        this.createPlayer();
+    }
+
+    @Override
+    protected List<SourceDataset> extract()
+    {
+        return null;
+    }
+
+    @Override
+    protected List<PlayerTarget> transform(List<SourceDataset> extractedData)
+    {
+        return null;
+    }
+
+    @Override
+    protected void load(List<PlayerTarget> transformedData)
+    {
+    }
+
+    private void createPlayer()
+    {
+        var sql = "insert into player values (?, ?)";
+
+        StatementPreparerLoader<PlayerTarget> statementPreparerLoader =  (preparedStatement, data) -> {
+            preparedStatement.setInt(1, data.getPlayerId());
+            preparedStatement.setString(2, data.getPlayerName());
+        };
+
+        var transformedData = List.of(new PlayerTarget(0, "Dummy Name"));
+
+        new Loader<>(this.target, statementPreparerLoader, transformedData, sql).doLoad();
+    }
+}

src/main/java/migration/ThesisMigration.java

@@ -0,0 +1,59 @@
+package migration;
+
+import org.apache.log4j.Logger;
+import utils.ConnectionHelper;
+import utils.DatabaseInformation;
+import utils.DatabaseType;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ThesisMigration
+{
+    private static Logger log = Logger.getLogger(ThesisMigration.class.getName());
+
+    private List<ETL> migrations;
+
+    private Connection mariadb;
+    private Connection mysql;
+    private Connection postgresql;
+
+    public ThesisMigration()
+    {
+        log.info("\n--- Creating connections ---");
+        final var mariaInfo = new DatabaseInformation("localhost", "sourcedb1", "test", "test", 25003);
+        this.mariadb = new ConnectionHelper(DatabaseType.MARIADB, mariaInfo).createConnection();
+
+        final var mysqlInfo = new DatabaseInformation("localhost", "sourcedb2", "test", "test", 25002);
+        this.mysql = new ConnectionHelper(DatabaseType.MYSQL, mysqlInfo).createConnection();
+
+        final var postgresInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 25001);
+        this.postgresql= new ConnectionHelper(DatabaseType.POSTGRESQL, postgresInfo).createConnection();
+
+        this.migrations = new ArrayList<>();
+        this.migrations.add(new PlayerMigration(null, this.postgresql));
+        this.migrations.add(new AbilityMigration(this.mysql, this.postgresql));
+        this.migrations.add(new PlayerAbilitiesMigration(this.postgresql, this.postgresql));
+        this.migrations.add(new CharacterMigration(this.mariadb, this.postgresql));
+    }
+
+    public void executeMigrations()
+    {
+        log.info("\n----- Starting migrations for each migration step -----\n");
+        this.migrations.forEach(ETL::migrate);
+
+        try
+        {
+            log.info("\n--- Closing connections ---\n");
+            this.mariadb.close();
+            this.mysql.close();
+            this.postgresql.close();
+        }
+        catch (SQLException e)
+        {
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/utils/ConnectionHelper.java

@@ -1,12 +1,16 @@
 package utils;
 
+import migration.ThesisMigration;
+import org.apache.log4j.Logger;
+
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.sql.*;
-import java.util.Properties;
 
 public class ConnectionHelper
 {
+    private static Logger log = Logger.getLogger(ConnectionHelper.class.getName());
+
     private final DatabaseType databaseType;
     private final String user;
     private final String password;
@@ -46,20 +50,23 @@ public class ConnectionHelper
             e.printStackTrace();
         }
 
-        Connection c = null;
+        Connection connection = null;
 
         try
         {
-            c = DriverManager.getConnection(this.uri.toString(), this.user, this.password);
+            log.info(String.format("Trying to connect to %s", this.uri.toString()));
+            connection = DriverManager.getConnection(this.uri.toString(), this.user, this.password);
         }
         catch (final SQLException e)
         {
-            System.out.println("SQLException: " + e.getMessage());
-            System.out.println("SQLState: " + e.getSQLState());
-            System.out.println("VendorError: " + e.getErrorCode());
+            log.error(String.format("""
+                    SQLException: %s;
+                    SQLState: %s;
+                    VendorError:%s""",
+                    e.getMessage(), e.getSQLState(), e.getErrorCode()));
             e.printStackTrace();
         }
 
-        return c;
+        return connection;
     }
 }

src/main/resources/log4j.properties

@@ -0,0 +1,8 @@
+# Root logger option
+log4j.rootLogger=ALL, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n