diff --git a/pom.xml b/pom.xml index 201561d..c242fae 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,12 @@ postgresql 42.2.9 + + log4j + log4j + 1.2.17 + + \ No newline at end of file diff --git a/src/main/java/Main.java b/src/main/java/Main.java index 1434645..7b5c09a 100644 --- a/src/main/java/Main.java +++ b/src/main/java/Main.java @@ -1,17 +1,21 @@ import data.source.PersonSource; import data.target.CharacterTarget; import etl.*; +import migration.ThesisMigration; +import org.apache.log4j.Logger; import utils.*; import java.util.*; public class Main { + private static Logger log = Logger.getLogger(Main.class.getName()); + public static void main(final String[] args) { - var p = testExtract(); - var pt = testTransform(p); - testLoad(pt); + log.info("------------- Starting Migration -------------"); + new ThesisMigration().executeMigrations(); + log.info("------------- Migration Finished -------------"); } private static List testExtract() diff --git a/src/main/java/data/source/PlayerAbilitiesSource.java b/src/main/java/data/source/PlayerAbilitiesSource.java new file mode 100644 index 0000000..04287e0 --- /dev/null +++ b/src/main/java/data/source/PlayerAbilitiesSource.java @@ -0,0 +1,32 @@ +package data.source; + +import data.SourceDataset; + +public class PlayerAbilitiesSource implements SourceDataset +{ + private int playerId; + private int abilityId; + + public PlayerAbilitiesSource(int playerId, int abilityId) + { + this.playerId = playerId; + this.abilityId = abilityId; + } + + public int getPlayerId() + { + return playerId; + } + + public int getAbilityId() + { + return abilityId; + } + + @Override + public String toString() + { + return String.format("PlayerAbility [ %d, %d ]", this.playerId, this.abilityId); + } + +} diff --git a/src/main/java/etl/Extractor.java b/src/main/java/etl/Extractor.java index ec4e181..dcb4a7d 100644 --- a/src/main/java/etl/Extractor.java +++ b/src/main/java/etl/Extractor.java @@ -1,6 +1,7 @@ package etl; import data.SourceDataset; +import org.apache.log4j.Logger; import utils.DataStorer; import utils.StatementPreparerExtractor; @@ -10,6 +11,8 @@ import java.util.List; public class Extractor { + private static Logger log = Logger.getLogger(Extractor.class.getName()); + private Connection sourceDatabase; private DataStorer dataStorer; private StatementPreparerExtractor statementPreparer; @@ -31,7 +34,7 @@ public class Extractor var preparedStatement = this.sourceDatabase.prepareStatement(sql); this.statementPreparer.doPrepare(preparedStatement); var resultSet = preparedStatement.executeQuery(); - this.sourceDatabase.close(); + log.info(String.format("--- Data extracted with '%s' ---", this.sql)); return this.dataStorer.doStore(resultSet); } catch (SQLException e) diff --git a/src/main/java/etl/Loader.java b/src/main/java/etl/Loader.java index fd8b791..65c449d 100644 --- a/src/main/java/etl/Loader.java +++ b/src/main/java/etl/Loader.java @@ -2,6 +2,7 @@ package etl; import data.SourceDataset; import data.TargetDataset; +import org.apache.log4j.Logger; import utils.StatementPreparerLoader; import java.sql.Connection; @@ -10,6 +11,8 @@ import java.util.List; public class Loader { + private static Logger log = Logger.getLogger(Loader.class.getName()); + private Connection targetDatabase; private StatementPreparerLoader statementPreparerLoader; private List transformedData; @@ -30,12 +33,12 @@ public class Loader { for (T transformedDatum : this.transformedData) { + log.info(String.format("Load data into target: %s", transformedDatum)); var preparedStatement = this.targetDatabase.prepareStatement(this.sql); this.statementPreparerLoader.doPrepare(preparedStatement, transformedDatum); preparedStatement.executeUpdate(); } - - this.targetDatabase.close(); + log.info(String.format("--- Data loaded into target with '%s' ---", this.sql)); } catch (SQLException e) { diff --git a/src/main/java/etl/Transformer.java b/src/main/java/etl/Transformer.java index bffde1d..fef0322 100644 --- a/src/main/java/etl/Transformer.java +++ b/src/main/java/etl/Transformer.java @@ -2,6 +2,7 @@ package etl; import data.SourceDataset; import data.TargetDataset; +import org.apache.log4j.Logger; import utils.DataTransformer; import java.util.ArrayList; @@ -9,6 +10,7 @@ import java.util.List; public class Transformer { + private static Logger log = Logger.getLogger(Transformer.class.getName()); private DataTransformer transformer; private List extractedData; @@ -24,8 +26,10 @@ public class Transformer var transformed = new ArrayList(); for (T dataset : this.extractedData) { + log.info(String.format("Transform data: %s", dataset)); transformed.add(this.transformer.transform(dataset)); } + log.info("--- Data transformed ---"); return transformed; } } diff --git a/src/main/java/migration/AbilityMigration.java b/src/main/java/migration/AbilityMigration.java new file mode 100644 index 0000000..7525912 --- /dev/null +++ b/src/main/java/migration/AbilityMigration.java @@ -0,0 +1,76 @@ +package migration; + +import data.source.AbilitiesSource; +import data.source.PersonSource; +import data.target.AbilityTarget; +import data.target.CharacterTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.*; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + +public class AbilityMigration extends ETL +{ + private int abilityId; + + public AbilityMigration(Connection source, Connection target) + { + super(source, target); + this.abilityId = 0; + } + + @Override + protected List extract() + { + DataStorer dataStorer = (resultSet) -> { + var extractedData = new ArrayList(); + while (resultSet.next()) + { + extractedData.add(new AbilitiesSource( + resultSet.getString("name"), + resultSet.getString("description"), + resultSet.getInt("level") + )); + } + return extractedData; + }; + + var sql = "select * from abilities;"; + StatementPreparerExtractor statementPreparer = (preparedStatement) -> {}; + + return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract(); + } + + @Override + protected List transform(List extractedData) + { + DataTransformer transformer = + (dataset) -> new AbilityTarget( + abilityId++, + dataset.getName(), + dataset.getDescription(), + dataset.getLevel() + ); + + return new Transformer<>(transformer, extractedData).doTransform(); + } + + @Override + protected void load(List transformedData) + { + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getAbilityId()); + preparedStatement.setString(2, data.getAbilityName()); + preparedStatement.setString(3, data.getAbilityDescription()); + preparedStatement.setInt(4, data.getAbilityLevel()); + }; + + var sql = "insert into ability values (?, ?, ?, ?)"; + + new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/migration/CharacterMigration.java b/src/main/java/migration/CharacterMigration.java new file mode 100644 index 0000000..bc11304 --- /dev/null +++ b/src/main/java/migration/CharacterMigration.java @@ -0,0 +1,74 @@ +package migration; + +import data.source.AbilitiesSource; +import data.source.PersonSource; +import data.target.AbilityTarget; +import data.target.CharacterTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.DataStorer; +import utils.DataTransformer; +import utils.StatementPreparerExtractor; +import utils.StatementPreparerLoader; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + +public class CharacterMigration extends ETL +{ + public CharacterMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + protected List extract() + { + DataStorer dataStorer = (resultSet) -> { + var extractedData = new ArrayList(); + while (resultSet.next()) + { + extractedData.add(new PersonSource( + resultSet.getInt("personId"), + resultSet.getString("name"), + resultSet.getBoolean("mortal") + )); + } + return extractedData; + }; + + var sql = "select * from person;"; + StatementPreparerExtractor statementPreparer = (preparedStatement) -> {}; + + return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract(); + } + + @Override + protected List transform(List extractedData) + { + DataTransformer transformer = + (dataset) -> new CharacterTarget( + dataset.getPersonId(), + dataset.getName(), + dataset.isMortal() + ); + + return new Transformer<>(transformer, extractedData).doTransform(); + } + + @Override + protected void load(List transformedData) + { + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getPersonId()); + preparedStatement.setString(2, data.getName()); + preparedStatement.setBoolean(3, data.isMortal()); + }; + + var sql = "insert into character values (?, ?, ?)"; + + new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/migration/ETL.java b/src/main/java/migration/ETL.java new file mode 100644 index 0000000..728f849 --- /dev/null +++ b/src/main/java/migration/ETL.java @@ -0,0 +1,35 @@ +package migration; + +import data.SourceDataset; +import data.TargetDataset; +import data.source.PersonSource; +import org.apache.log4j.Logger; + +import java.sql.Connection; +import java.util.List; + +public abstract class ETL +{ + protected static Logger log = Logger.getLogger(ETL.class.getName()); + + protected Connection source; + protected Connection target; + + public ETL(Connection source, Connection target) + { + this.source = source; + this.target = target; + } + + public void migrate() + { + log.info(String.format("--- Migration: %s ---", this.getClass().getName())); + var extractedData = this.extract(); + var transformedData = this.transform(extractedData); + this.load(transformedData); + } + + protected abstract List extract(); + protected abstract List transform(List extractedData); + protected abstract void load(List transformedData); +} diff --git a/src/main/java/migration/GameobjectMigration.java b/src/main/java/migration/GameobjectMigration.java new file mode 100644 index 0000000..0591bea --- /dev/null +++ b/src/main/java/migration/GameobjectMigration.java @@ -0,0 +1,33 @@ +package migration; + +import data.source.GameobjectSource; +import data.target.GameobjectTarget; + +import java.sql.Connection; +import java.util.List; + +public class GameobjectMigration extends ETL +{ + public GameobjectMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + protected List extract() + { + return null; + } + + @Override + protected List transform(List extractedData) + { + return null; + } + + @Override + protected void load(List transformedData) + { + + } +} diff --git a/src/main/java/migration/PlayerAbilitiesMigration.java b/src/main/java/migration/PlayerAbilitiesMigration.java new file mode 100644 index 0000000..8337d1e --- /dev/null +++ b/src/main/java/migration/PlayerAbilitiesMigration.java @@ -0,0 +1,72 @@ +package migration; + +import data.SourceDataset; +import data.source.AbilitiesSource; +import data.source.PlayerAbilitiesSource; +import data.target.AbilityTarget; +import data.target.PlayerAbilitiesTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.DataStorer; +import utils.DataTransformer; +import utils.StatementPreparerExtractor; +import utils.StatementPreparerLoader; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; + +public class PlayerAbilitiesMigration extends ETL +{ + public PlayerAbilitiesMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + protected List extract() + { + DataStorer dataStorer = (resultSet) -> { + var extractedData = new ArrayList(); + while (resultSet.next()) + { + extractedData.add(new PlayerAbilitiesSource( + resultSet.getInt("playerId"), + resultSet.getInt("abilityId") + )); + } + return extractedData; + }; + + var sql = "select * from player p join ability a on p.playerId = ?;"; + StatementPreparerExtractor statementPreparer = (preparedStatement) -> { + preparedStatement.setInt(1, 0); + }; + + return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract(); + } + + @Override + protected List transform(List extractedData) + { + DataTransformer transformer = + (dataset) -> new PlayerAbilitiesTarget(dataset.getPlayerId(), dataset.getAbilityId()); + + return new Transformer<>(transformer, extractedData).doTransform(); + } + + @Override + protected void load(List transformedData) + { + + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getPlayerId()); + preparedStatement.setInt(2, data.getAbilityId()); + }; + + var sql = "insert into playerAbilities values (?, ?)"; + + new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/migration/PlayerMigration.java b/src/main/java/migration/PlayerMigration.java new file mode 100644 index 0000000..708eb48 --- /dev/null +++ b/src/main/java/migration/PlayerMigration.java @@ -0,0 +1,57 @@ +package migration; + +import data.SourceDataset; +import data.target.PlayerTarget; +import etl.Loader; +import utils.ConnectionHelper; +import utils.DatabaseInformation; +import utils.DatabaseType; +import utils.StatementPreparerLoader; + +import java.sql.Connection; +import java.util.List; + +public class PlayerMigration extends ETL +{ + public PlayerMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + public void migrate() + { + this.createPlayer(); + } + + @Override + protected List extract() + { + return null; + } + + @Override + protected List transform(List extractedData) + { + return null; + } + + @Override + protected void load(List transformedData) + { + } + + private void createPlayer() + { + var sql = "insert into player values (?, ?)"; + + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getPlayerId()); + preparedStatement.setString(2, data.getPlayerName()); + }; + + var transformedData = List.of(new PlayerTarget(0, "Dummy Name")); + + new Loader<>(this.target, statementPreparerLoader, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/migration/ThesisMigration.java b/src/main/java/migration/ThesisMigration.java new file mode 100644 index 0000000..966392f --- /dev/null +++ b/src/main/java/migration/ThesisMigration.java @@ -0,0 +1,59 @@ +package migration; + +import org.apache.log4j.Logger; +import utils.ConnectionHelper; +import utils.DatabaseInformation; +import utils.DatabaseType; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class ThesisMigration +{ + private static Logger log = Logger.getLogger(ThesisMigration.class.getName()); + + private List migrations; + + private Connection mariadb; + private Connection mysql; + private Connection postgresql; + + public ThesisMigration() + { + log.info("\n--- Creating connections ---"); + final var mariaInfo = new DatabaseInformation("localhost", "sourcedb1", "test", "test", 25003); + this.mariadb = new ConnectionHelper(DatabaseType.MARIADB, mariaInfo).createConnection(); + + final var mysqlInfo = new DatabaseInformation("localhost", "sourcedb2", "test", "test", 25002); + this.mysql = new ConnectionHelper(DatabaseType.MYSQL, mysqlInfo).createConnection(); + + final var postgresInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 25001); + this.postgresql= new ConnectionHelper(DatabaseType.POSTGRESQL, postgresInfo).createConnection(); + + this.migrations = new ArrayList<>(); + this.migrations.add(new PlayerMigration(null, this.postgresql)); + this.migrations.add(new AbilityMigration(this.mysql, this.postgresql)); + this.migrations.add(new PlayerAbilitiesMigration(this.postgresql, this.postgresql)); + this.migrations.add(new CharacterMigration(this.mariadb, this.postgresql)); + } + + public void executeMigrations() + { + log.info("\n----- Starting migrations for each migration step -----\n"); + this.migrations.forEach(ETL::migrate); + + try + { + log.info("\n--- Closing connections ---\n"); + this.mariadb.close(); + this.mysql.close(); + this.postgresql.close(); + } + catch (SQLException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/utils/ConnectionHelper.java b/src/main/java/utils/ConnectionHelper.java index 61dd99c..c1407ee 100644 --- a/src/main/java/utils/ConnectionHelper.java +++ b/src/main/java/utils/ConnectionHelper.java @@ -1,12 +1,16 @@ package utils; +import migration.ThesisMigration; +import org.apache.log4j.Logger; + import java.net.URI; import java.net.URISyntaxException; import java.sql.*; -import java.util.Properties; public class ConnectionHelper { + private static Logger log = Logger.getLogger(ConnectionHelper.class.getName()); + private final DatabaseType databaseType; private final String user; private final String password; @@ -46,20 +50,23 @@ public class ConnectionHelper e.printStackTrace(); } - Connection c = null; + Connection connection = null; try { - c = DriverManager.getConnection(this.uri.toString(), this.user, this.password); + log.info(String.format("Trying to connect to %s", this.uri.toString())); + connection = DriverManager.getConnection(this.uri.toString(), this.user, this.password); } catch (final SQLException e) { - System.out.println("SQLException: " + e.getMessage()); - System.out.println("SQLState: " + e.getSQLState()); - System.out.println("VendorError: " + e.getErrorCode()); + log.error(String.format(""" + SQLException: %s; + SQLState: %s; + VendorError:%s""", + e.getMessage(), e.getSQLState(), e.getErrorCode())); e.printStackTrace(); } - return c; + return connection; } } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..12f358e --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=ALL, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file