From 780bad6dbd4d1875ef21419979dcf436c75bb87d Mon Sep 17 00:00:00 2001 From: Niklas Birk Date: Thu, 6 Feb 2020 20:37:29 +0100 Subject: [PATCH] Added few Migrations --- .../data/source/QuestParticipationSource.java | 31 ------- src/main/java/data/source/QuestSource.java | 49 +++++++++- src/main/java/data/target/AbilityTarget.java | 8 +- src/main/java/data/target/QuestTarget.java | 20 +++- src/main/java/migration/AbilityMigration.java | 22 +++-- src/main/java/migration/ETL.java | 10 ++ .../java/migration/GameobjectMigration.java | 45 ++++++++- src/main/java/migration/ModMigration.java | 33 +++++++ src/main/java/migration/QuestMigration.java | 93 +++++++++++++++++++ src/main/java/migration/ThesisMigration.java | 13 +++ src/main/java/utils/ConnectionHelper.java | 1 - 11 files changed, 271 insertions(+), 54 deletions(-) delete mode 100644 src/main/java/data/source/QuestParticipationSource.java create mode 100644 src/main/java/migration/ModMigration.java create mode 100644 src/main/java/migration/QuestMigration.java diff --git a/src/main/java/data/source/QuestParticipationSource.java b/src/main/java/data/source/QuestParticipationSource.java deleted file mode 100644 index f00827d..0000000 --- a/src/main/java/data/source/QuestParticipationSource.java +++ /dev/null @@ -1,31 +0,0 @@ -package data.source; - -import data.SourceDataset; - -public class QuestParticipationSource implements SourceDataset -{ - private int questId; - private int personId; - - public QuestParticipationSource(int questId, int personId) - { - this.questId = questId; - this.personId = personId; - } - - public int getPersonId() - { - return personId; - } - - public int getQuestId() - { - return questId; - } - - @Override - public String toString() - { - return String.format("QuestParticipation { %d, %d }", this.questId, this.personId); - } -} diff --git a/src/main/java/data/source/QuestSource.java b/src/main/java/data/source/QuestSource.java index 2f4daad..826f27e 100644 --- a/src/main/java/data/source/QuestSource.java +++ b/src/main/java/data/source/QuestSource.java @@ -2,17 +2,23 @@ package data.source; import data.SourceDataset; +import java.sql.Clob; +import java.sql.SQLException; +import java.util.Objects; + public class QuestSource implements SourceDataset { private int questId; private String name; - private String dialogue; + private Clob dialogue; + private int personId; - public QuestSource(int questId, String name, String dialogue) + public QuestSource(int questId, String name, Clob dialogue, int personId) { this.questId = questId; this.name = name; this.dialogue = dialogue; + this.personId = personId; } public int getQuestId() @@ -25,14 +31,49 @@ public class QuestSource implements SourceDataset return name; } - public String getDialogue() + public Clob getDialogue() { return dialogue; } + public int getPersonId() + { + return personId; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + { + return true; + } + if (o == null || getClass() != o.getClass()) + { + return false; + } + QuestSource that = (QuestSource) o; + return questId == that.questId; + } + + @Override + public int hashCode() + { + return Objects.hash(questId); + } + @Override public String toString() { - return String.format("Quest { %d, %s, %s... }", this.questId, this.name, this.dialogue.substring(0, 10)); + try + { + return String.format("Quest { %d, %s, %s..., %d }", this.questId, this.name, this.dialogue.getSubString(1, 10), this.personId); + } + catch (SQLException e) + { + e.printStackTrace(); + } + + return null; } } diff --git a/src/main/java/data/target/AbilityTarget.java b/src/main/java/data/target/AbilityTarget.java index d20f312..0ee02fe 100644 --- a/src/main/java/data/target/AbilityTarget.java +++ b/src/main/java/data/target/AbilityTarget.java @@ -7,9 +7,9 @@ public class AbilityTarget implements TargetDataset private int abilityId; private String abilityName; private String abilityDescription; - private int abilityLevel; + private float abilityLevel; - public AbilityTarget(int abilityId, String abilityName, String abilityDescription, int abilityLevel) + public AbilityTarget(int abilityId, String abilityName, String abilityDescription, float abilityLevel) { this.abilityId = abilityId; this.abilityName = abilityName; @@ -32,7 +32,7 @@ public class AbilityTarget implements TargetDataset return abilityDescription; } - public int getAbilityLevel() + public float getAbilityLevel() { return abilityLevel; } @@ -40,7 +40,7 @@ public class AbilityTarget implements TargetDataset @Override public String toString() { - return String.format("Ability [ %d, %s, %s, %d ]", this.abilityId, this.abilityName, this.abilityDescription, this.abilityLevel); + return String.format("Ability [ %d, %s, %s, %f ]", this.abilityId, this.abilityName, this.abilityDescription, this.abilityLevel); } } diff --git a/src/main/java/data/target/QuestTarget.java b/src/main/java/data/target/QuestTarget.java index 0a1bbbd..d0bcacb 100644 --- a/src/main/java/data/target/QuestTarget.java +++ b/src/main/java/data/target/QuestTarget.java @@ -2,14 +2,17 @@ package data.target; import data.TargetDataset; +import java.sql.Clob; +import java.sql.SQLException; + public class QuestTarget implements TargetDataset { private int questId; private String name; private String involvedCharacters; - private String dialogue; + private Clob dialogue; - public QuestTarget(int questId, String name, String involvedCharacters, String dialogue) + public QuestTarget(int questId, String name, String involvedCharacters, Clob dialogue) { this.questId = questId; this.name = name; @@ -32,7 +35,7 @@ public class QuestTarget implements TargetDataset return involvedCharacters; } - public String getDialogue() + public Clob getDialogue() { return dialogue; } @@ -40,6 +43,15 @@ public class QuestTarget implements TargetDataset @Override public String toString() { - return String.format("Quest [ %d, %s, %s, %s... ]", this.questId, this.name, this.involvedCharacters, this.dialogue.substring(0, 10)); + try + { + return String.format("Quest [ %d, %s, { %s }, %s... ]", this.questId, this.name, this.involvedCharacters, this.dialogue.getSubString(1, 10)); + } + catch (SQLException e) + { + e.printStackTrace(); + } + + return null; } } diff --git a/src/main/java/migration/AbilityMigration.java b/src/main/java/migration/AbilityMigration.java index 7525912..96ce99d 100644 --- a/src/main/java/migration/AbilityMigration.java +++ b/src/main/java/migration/AbilityMigration.java @@ -1,5 +1,6 @@ package migration; +import com.mysql.cj.exceptions.NumberOutOfRange; import data.source.AbilitiesSource; import data.source.PersonSource; import data.target.AbilityTarget; @@ -48,13 +49,18 @@ public class AbilityMigration extends ETL @Override protected List transform(List extractedData) { - DataTransformer transformer = - (dataset) -> new AbilityTarget( - abilityId++, - dataset.getName(), - dataset.getDescription(), - dataset.getLevel() - ); + DataTransformer transformer = (dataset) -> { + if (dataset.getLevel() < 0 || dataset.getLevel() > 100) + log.error("level-value out of range", + new NumberOutOfRange("level must be within 0 an 100 (both inclusive)")); + var levelCorrection = 100f; + return new AbilityTarget( + abilityId++, + dataset.getName(), + dataset.getDescription(), + dataset.getLevel() / levelCorrection + ); + }; return new Transformer<>(transformer, extractedData).doTransform(); } @@ -66,7 +72,7 @@ public class AbilityMigration extends ETL preparedStatement.setInt(1, data.getAbilityId()); preparedStatement.setString(2, data.getAbilityName()); preparedStatement.setString(3, data.getAbilityDescription()); - preparedStatement.setInt(4, data.getAbilityLevel()); + preparedStatement.setFloat(4, data.getAbilityLevel()); }; var sql = "insert into ability values (?, ?, ?, ?)"; diff --git a/src/main/java/migration/ETL.java b/src/main/java/migration/ETL.java index 728f849..bd7fce1 100644 --- a/src/main/java/migration/ETL.java +++ b/src/main/java/migration/ETL.java @@ -6,6 +6,7 @@ import data.source.PersonSource; import org.apache.log4j.Logger; import java.sql.Connection; +import java.sql.SQLException; import java.util.List; public abstract class ETL @@ -27,6 +28,15 @@ public abstract class ETL var extractedData = this.extract(); var transformedData = this.transform(extractedData); this.load(transformedData); + + try + { + this.target.commit(); + } + catch (SQLException e) + { + e.printStackTrace(); + } } protected abstract List extract(); diff --git a/src/main/java/migration/GameobjectMigration.java b/src/main/java/migration/GameobjectMigration.java index 0591bea..f225381 100644 --- a/src/main/java/migration/GameobjectMigration.java +++ b/src/main/java/migration/GameobjectMigration.java @@ -1,9 +1,20 @@ package migration; +import com.mysql.cj.exceptions.NumberOutOfRange; +import data.source.AbilitiesSource; import data.source.GameobjectSource; +import data.target.AbilityTarget; import data.target.GameobjectTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.DataStorer; +import utils.DataTransformer; +import utils.StatementPreparerExtractor; +import utils.StatementPreparerLoader; import java.sql.Connection; +import java.util.ArrayList; import java.util.List; public class GameobjectMigration extends ETL @@ -16,18 +27,48 @@ public class GameobjectMigration extends ETL @Override protected List extract() { - return null; + DataStorer dataStorer = (resultSet) -> { + var extractedData = new ArrayList(); + while (resultSet.next()) + { + extractedData.add(new GameobjectSource( + resultSet.getInt("objectId"), + resultSet.getString("name"), + resultSet.getString("description") + )); + } + return extractedData; + }; + + var sql = "select * from gameobject;"; + StatementPreparerExtractor statementPreparer = (preparedStatement) -> {}; + + return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract(); } @Override protected List transform(List extractedData) { - return null; + DataTransformer transformer = (dataset) -> new GameobjectTarget( + dataset.getObjectId(), + dataset.getName(), + dataset.getDescription() + ); + + return new Transformer<>(transformer, extractedData).doTransform(); } @Override protected void load(List transformedData) { + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getObjectId()); + preparedStatement.setString(2, data.getName()); + preparedStatement.setString(3, data.getDescription()); + }; + var sql = "insert into gameobject values (?, ?, ?)"; + + new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad(); } } diff --git a/src/main/java/migration/ModMigration.java b/src/main/java/migration/ModMigration.java new file mode 100644 index 0000000..3394dfc --- /dev/null +++ b/src/main/java/migration/ModMigration.java @@ -0,0 +1,33 @@ +package migration; + +import data.source.ModSource; +import data.target.ModTarget; + +import java.sql.Connection; +import java.util.List; + +public class ModMigration extends ETL +{ + public ModMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + protected List extract() + { + return null; + } + + @Override + protected List transform(List extractedData) + { + return null; + } + + @Override + protected void load(List transformedData) + { + + } +} diff --git a/src/main/java/migration/QuestMigration.java b/src/main/java/migration/QuestMigration.java new file mode 100644 index 0000000..9afc279 --- /dev/null +++ b/src/main/java/migration/QuestMigration.java @@ -0,0 +1,93 @@ +package migration; + +import data.source.QuestSource; +import data.target.QuestTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.DataStorer; +import utils.DataTransformer; +import utils.StatementPreparerExtractor; +import utils.StatementPreparerLoader; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +public class QuestMigration extends ETL +{ + public QuestMigration(Connection source, Connection target) + { + super(source, target); + } + + @Override + protected List extract() + { + DataStorer dataStorer = (resultSet) -> { + var extractedData = new ArrayList(); + while (resultSet.next()) + { + extractedData.add(new QuestSource( + resultSet.getInt("questId"), + resultSet.getString("name"), + resultSet.getClob("dialogue"), + resultSet.getInt("personId"))); + } + return extractedData; + }; + + var sql = "select * from quest q join questParticipation qp on q.questId = qp.questId;"; + StatementPreparerExtractor statementPreparer = (preparedStatement) -> {}; + + return new Extractor<>(super.source, dataStorer, statementPreparer, sql).doExtract(); + } + + @Override + protected List transform(List extractedData) + { + var map = new HashMap(); + + for (QuestSource extractedDatum : extractedData) + { + var key = extractedDatum.getQuestId(); + if (!map.containsKey(key)) + { + map.put(key, String.valueOf(extractedDatum.getPersonId())); + } + else + { + var involvedPersons = String.format("%s, %s", map.get(key), extractedDatum.getPersonId()); + map.replace(key, involvedPersons); + } + } + + extractedData = extractedData.stream().distinct().collect(Collectors.toList()); + + DataTransformer transformer = (dataset) -> new QuestTarget( + dataset.getQuestId(), + dataset.getName(), + map.get(dataset.getQuestId()), + dataset.getDialogue() + ); + + return new Transformer<>(transformer, extractedData).doTransform(); + } + + @Override + protected void load(List transformedData) + { + StatementPreparerLoader statementPreparerLoader = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getQuestId()); + preparedStatement.setString(2, data.getName()); + preparedStatement.setString(3, data.getInvolvedCharacters()); + preparedStatement.setClob(4, data.getDialogue()); + }; + + var sql = "insert into quest values (?, ?, ?, ?)"; + + new Loader<>(super.target, statementPreparerLoader, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/migration/ThesisMigration.java b/src/main/java/migration/ThesisMigration.java index 966392f..d2adaf1 100644 --- a/src/main/java/migration/ThesisMigration.java +++ b/src/main/java/migration/ThesisMigration.java @@ -32,11 +32,24 @@ public class ThesisMigration final var postgresInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 25001); this.postgresql= new ConnectionHelper(DatabaseType.POSTGRESQL, postgresInfo).createConnection(); + try + { + this.mariadb.setAutoCommit(false); + this.mysql.setAutoCommit(false); + this.postgresql.setAutoCommit(false); + } + catch (SQLException e) + { + e.printStackTrace(); + } + this.migrations = new ArrayList<>(); this.migrations.add(new PlayerMigration(null, this.postgresql)); this.migrations.add(new AbilityMigration(this.mysql, this.postgresql)); this.migrations.add(new PlayerAbilitiesMigration(this.postgresql, this.postgresql)); this.migrations.add(new CharacterMigration(this.mariadb, this.postgresql)); + this.migrations.add(new GameobjectMigration(this.mariadb, this.postgresql)); + this.migrations.add(new QuestMigration(this.mariadb, this.postgresql)); } public void executeMigrations() diff --git a/src/main/java/utils/ConnectionHelper.java b/src/main/java/utils/ConnectionHelper.java index c1407ee..81155d0 100644 --- a/src/main/java/utils/ConnectionHelper.java +++ b/src/main/java/utils/ConnectionHelper.java @@ -64,7 +64,6 @@ public class ConnectionHelper SQLState: %s; VendorError:%s""", e.getMessage(), e.getSQLState(), e.getErrorCode())); - e.printStackTrace(); } return connection;