Added all datasets

This commit is contained in:
Niklas Birk 2020-02-06 17:41:22 +01:00
parent 6b23c32c6d
commit c14e94d498
32 changed files with 771 additions and 69 deletions

View File

@ -1,68 +1,61 @@
import data.source.PersonSource; import data.source.PersonSource;
import data.target.PersonTarget; import data.target.CharacterTarget;
import etl.Extractor; import etl.*;
import etl.Loader;
import etl.Transformer;
import utils.*; import utils.*;
import java.sql.Connection; import java.util.*;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.time.Period;
import java.util.ArrayList;
import java.util.List;
public class Main public class Main
{ {
public static void main(final String[] args) throws SQLException public static void main(final String[] args)
{ {
// mysqlDb();
// postgresqlDb();
// mariaDb();
var p = testExtract(); var p = testExtract();
var pt = testTransform(p); var pt = testTransform(p);
testLoad(pt); testLoad(pt);
} }
private static List<PersonSource> testExtract() throws SQLException private static List<PersonSource> testExtract()
{ {
final var dbInfo = new DatabaseInformation("localhost", "sourcedb1", "test", "test", 5435); final var dbInfo = new DatabaseInformation("localhost", "sourcedb1", "test", "test", 25003);
final var connection = new ConnectionHelper(DatabaseType.MARIADB, dbInfo).createConnection(); final var connection = new ConnectionHelper(DatabaseType.MARIADB, dbInfo).createConnection();
DataStorer<PersonSource> personSourceDataStorer = (rs) -> { DataStorer<PersonSource> personSourceDataStorer = (rs) -> {
var persons = new ArrayList<PersonSource>(); var persons = new ArrayList<PersonSource>();
while (rs.next()) while (rs.next())
{ {
persons.add(new PersonSource(rs.getInt("personId"), persons.add(new PersonSource(
rs.getInt("personId"),
rs.getString("name"), rs.getString("name"),
rs.getBoolean("mortal"))); rs.getBoolean("mortal"))
);
} }
return persons; return persons;
}; };
StatementPreparerExtractor statementPreparer = (preparedStatement) -> {
};
var sql = "select * from person;"; var sql = "select * from person;";
return new Extractor<>(connection, personSourceDataStorer, sql).doExtract(); return new Extractor<>(connection, personSourceDataStorer, statementPreparer, sql).doExtract();
} }
private static List<PersonTarget> testTransform(List<PersonSource> persons) private static List<CharacterTarget> testTransform(List<PersonSource> persons)
{ {
DataTransformer<PersonSource, PersonTarget> personTransformer = (personSource) -> { DataTransformer<PersonSource, CharacterTarget> personTransformer =
return new PersonTarget(personSource.getPersonId(), (personSource) -> new CharacterTarget(personSource.getPersonId(),
personSource.getName(), personSource.getName(),
personSource.isMortal()); personSource.isMortal());
};
return new Transformer<>(personTransformer, persons).doTransform(); return new Transformer<>(personTransformer, persons).doTransform();
} }
private static void testLoad(List<PersonTarget> transformedData) private static void testLoad(List<CharacterTarget> transformedData)
{ {
final var dbInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 5432); final var dbInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 25001);
final var connection = new ConnectionHelper(DatabaseType.POSTGRESQL, dbInfo).createConnection(); final var connection = new ConnectionHelper(DatabaseType.POSTGRESQL, dbInfo).createConnection();
StatementPreparer<PersonTarget> statementPreparer = (preparedStatement, data) -> { StatementPreparerLoader<CharacterTarget> statementPreparerLoader = (preparedStatement, data) -> {
preparedStatement.setInt(1, data.getPersonId()); preparedStatement.setInt(1, data.getPersonId());
preparedStatement.setString(2, data.getName()); preparedStatement.setString(2, data.getName());
preparedStatement.setBoolean(3, data.isMortal()); preparedStatement.setBoolean(3, data.isMortal());
@ -70,6 +63,6 @@ public class Main
var sql = "insert into person values (?, ?, ?)"; var sql = "insert into person values (?, ?, ?)";
new Loader<>(connection, statementPreparer, transformedData, sql).doLoad(); new Loader<>(connection, statementPreparerLoader, transformedData, sql).doLoad();
} }
} }

View File

@ -1,5 +0,0 @@
package data;
public interface Dataset
{
}

View File

@ -0,0 +1,5 @@
package data;
public interface SourceDataset
{
}

View File

@ -0,0 +1,5 @@
package data;
public interface TargetDataset
{
}

View File

@ -0,0 +1,38 @@
package data.source;
import data.SourceDataset;
public class AbilitiesSource implements SourceDataset
{
private String name;
private String description;
private int level;
public AbilitiesSource(String name, String description, int level)
{
this.name = name;
this.description = description;
this.level = level;
}
public String getName()
{
return name;
}
public String getDescription()
{
return description;
}
public int getLevel()
{
return level;
}
@Override
public String toString()
{
return String.format("Ability { %s, %s, %d }", this.name, this.description, this.level);
}
}

View File

@ -0,0 +1,31 @@
package data.source;
import data.SourceDataset;
public class ActiveQuestsSource implements SourceDataset
{
private int questId;
private int progress;
public ActiveQuestsSource(int questId, int progress)
{
this.questId = questId;
this.progress = progress;
}
public int getQuestId()
{
return questId;
}
public int getProgress()
{
return progress;
}
@Override
public String toString()
{
return String.format("ActiveQuest { %d, %d }", this.questId, this.progress);
}
}

View File

@ -0,0 +1,38 @@
package data.source;
import data.SourceDataset;
public class GameobjectSource implements SourceDataset
{
private int objectId;
private String name;
private String description;
public GameobjectSource(int objectId, String name, String description)
{
this.objectId = objectId;
this.name = name;
this.description = description;
}
public int getObjectId()
{
return objectId;
}
public String getName()
{
return name;
}
public String getDescription()
{
return description;
}
@Override
public String toString()
{
return String.format("Gameobject { %d, %s, %s }", this.objectId, this.name, this.description);
}
}

View File

@ -0,0 +1,24 @@
package data.source;
import data.SourceDataset;
public class InventorySource implements SourceDataset
{
private int objectId;
public InventorySource(int objectId)
{
this.objectId = objectId;
}
public int getObjectId()
{
return objectId;
}
@Override
public String toString()
{
return String.format("Inventory { %d }", this.objectId);
}
}

View File

@ -0,0 +1,48 @@
package data.source;
import data.SourceDataset;
import java.sql.Blob;
import java.time.LocalDate;
public class ModSource implements SourceDataset
{
private int modId;
private String name;
private LocalDate installationDate;
private Blob binary;
public ModSource(int modId, String name, LocalDate installationDate, Blob binary)
{
this.modId = modId;
this.name = name;
this.installationDate = installationDate;
this.binary = binary;
}
public int getModId()
{
return modId;
}
public String getName()
{
return name;
}
public LocalDate getInstallationDate()
{
return installationDate;
}
public Blob getBinary()
{
return binary;
}
@Override
public String toString()
{
return String.format("Mod { %d, %s, Binary: %s }", this.modId, this.name, this.binary);
}
}

View File

@ -0,0 +1,31 @@
package data.source;
import data.SourceDataset;
public class PersonInventorySource implements SourceDataset
{
private int personId;
private int objectId;
public PersonInventorySource(int personId, int objectId)
{
this.personId = personId;
this.objectId = objectId;
}
public int getPersonId()
{
return personId;
}
public int getObjectId()
{
return objectId;
}
@Override
public String toString()
{
return String.format("PersonInventory { %d, %d }", this.personId, this.objectId);
}
}

View File

@ -1,8 +1,8 @@
package data.source; package data.source;
import data.Dataset; import data.SourceDataset;
public class PersonSource implements Dataset public class PersonSource implements SourceDataset
{ {
private int personId; private int personId;
private String name; private String name;

View File

@ -0,0 +1,31 @@
package data.source;
import data.SourceDataset;
public class QuestParticipationSource implements SourceDataset
{
private int questId;
private int personId;
public QuestParticipationSource(int questId, int personId)
{
this.questId = questId;
this.personId = personId;
}
public int getPersonId()
{
return personId;
}
public int getQuestId()
{
return questId;
}
@Override
public String toString()
{
return String.format("QuestParticipation { %d, %d }", this.questId, this.personId);
}
}

View File

@ -0,0 +1,38 @@
package data.source;
import data.SourceDataset;
public class QuestSource implements SourceDataset
{
private int questId;
private String name;
private String dialogue;
public QuestSource(int questId, String name, String dialogue)
{
this.questId = questId;
this.name = name;
this.dialogue = dialogue;
}
public int getQuestId()
{
return questId;
}
public String getName()
{
return name;
}
public String getDialogue()
{
return dialogue;
}
@Override
public String toString()
{
return String.format("Quest { %d, %s, %s... }", this.questId, this.name, this.dialogue.substring(0, 10));
}
}

View File

@ -0,0 +1,31 @@
package data.source;
import data.SourceDataset;
public class RelationshipsSource implements SourceDataset
{
private int personId;
private int relationshipLevel;
public RelationshipsSource(int personId, int relationshipLevel)
{
this.personId = personId;
this.relationshipLevel = relationshipLevel;
}
public int getPersonId()
{
return personId;
}
public int getRelationshipLevel()
{
return relationshipLevel;
}
@Override
public String toString()
{
return String.format("Relationship { %d, %d }", this.personId, this.relationshipLevel);
}
}

View File

@ -0,0 +1,46 @@
package data.target;
import data.TargetDataset;
public class AbilityTarget implements TargetDataset
{
private int abilityId;
private String abilityName;
private String abilityDescription;
private int abilityLevel;
public AbilityTarget(int abilityId, String abilityName, String abilityDescription, int abilityLevel)
{
this.abilityId = abilityId;
this.abilityName = abilityName;
this.abilityDescription = abilityDescription;
this.abilityLevel = abilityLevel;
}
public int getAbilityId()
{
return abilityId;
}
public String getAbilityName()
{
return abilityName;
}
public String getAbilityDescription()
{
return abilityDescription;
}
public int getAbilityLevel()
{
return abilityLevel;
}
@Override
public String toString()
{
return String.format("Ability [ %d, %s, %s, %d ]", this.abilityId, this.abilityName, this.abilityDescription, this.abilityLevel);
}
}

View File

@ -0,0 +1,33 @@
package data.target;
import data.TargetDataset;
public class ActiveQuestsTarget implements TargetDataset
{
private int playerId;
private int questId;
private int questProgress;
public ActiveQuestsTarget(int playerId, int questId, int questProgress)
{
this.playerId = playerId;
this.questId = questId;
this.questProgress = questProgress;
}
public int getQuestId()
{
return questId;
}
public int getQuestProgress()
{
return questProgress;
}
@Override
public String toString()
{
return String.format("ActiveQuest [ %d, %d, %d ]", this.playerId, this.questId, this.questProgress);
}
}

View File

@ -0,0 +1,31 @@
package data.target;
import data.TargetDataset;
public class CharacterInventoryTarget implements TargetDataset
{
private int personId;
private int objectId;
public CharacterInventoryTarget(int personId, int objectId)
{
this.personId = personId;
this.objectId = objectId;
}
public int getPersonId()
{
return personId;
}
public int getObjectId()
{
return objectId;
}
@Override
public String toString()
{
return String.format("CharacterInventory [ %d, %d ]", this.personId, this.objectId);
}
}

View File

@ -1,14 +1,15 @@
package data.target; package data.target;
import data.Dataset; import data.SourceDataset;
import data.TargetDataset;
public class PersonTarget implements Dataset public class CharacterTarget implements TargetDataset
{ {
private int personId; private int personId;
private String name; private String name;
private boolean mortal; private boolean mortal;
public PersonTarget(int personId, String name, boolean mortal) public CharacterTarget(int personId, String name, boolean mortal)
{ {
this.personId = personId; this.personId = personId;
this.name = name; this.name = name;

View File

@ -0,0 +1,38 @@
package data.target;
import data.TargetDataset;
public class GameobjectTarget implements TargetDataset
{
private int objectId;
private String name;
private String description;
public GameobjectTarget(int objectId, String name, String description)
{
this.objectId = objectId;
this.name = name;
this.description = description;
}
public int getObjectId()
{
return objectId;
}
public String getName()
{
return name;
}
public String getDescription()
{
return description;
}
@Override
public String toString()
{
return String.format("Gameobject [ %d, %s, %s ]", this.objectId, this.name, this.description);
}
}

View File

@ -0,0 +1,38 @@
package data.target;
import data.TargetDataset;
public class InventoryTarget implements TargetDataset
{
private int playerId;
private int objectId;
private boolean stolen;
public InventoryTarget(int playerId, int objectId, boolean stolen)
{
this.playerId = playerId;
this.objectId = objectId;
this.stolen = stolen;
}
public int getPlayerId()
{
return playerId;
}
public int getObjectId()
{
return objectId;
}
public boolean isStolen()
{
return stolen;
}
@Override
public String toString()
{
return String.format("Inventory [ %d, %d, %s ]", this.playerId, this.objectId, this.stolen);
}
}

View File

@ -0,0 +1,49 @@
package data.target;
import data.TargetDataset;
import java.sql.Blob;
import java.time.LocalDate;
public class ModTarget implements TargetDataset
{
private int modId;
private String modName;
private LocalDate modInstallationDate;
private Blob modBinary;
public ModTarget(int modId, String modName, LocalDate modInstallationDate, Blob modBinary)
{
this.modId = modId;
this.modName = modName;
this.modInstallationDate = modInstallationDate;
this.modBinary = modBinary;
}
public int getModId()
{
return modId;
}
public String getName()
{
return modName;
}
public LocalDate getModInstallationDate()
{
return modInstallationDate;
}
public Blob getModBinary()
{
return modBinary;
}
@Override
public String toString()
{
return String.format("Mod { %d, %s, Binary: %s }", this.modId, this.modName, this.modBinary);
}
}

View File

@ -0,0 +1,31 @@
package data.target;
import data.TargetDataset;
public class PlayerAbilitiesTarget implements TargetDataset
{
private int playerId;
private int abilityId;
public PlayerAbilitiesTarget(int playerId, int abilityId)
{
this.playerId = playerId;
this.abilityId = abilityId;
}
public int getPlayerId()
{
return playerId;
}
public int getAbilityId()
{
return abilityId;
}
@Override
public String toString()
{
return String.format("PlayerAbility [ %d, %d ]", this.playerId, this.abilityId);
}
}

View File

@ -0,0 +1,31 @@
package data.target;
import data.TargetDataset;
public class PlayerTarget implements TargetDataset
{
private int playerId;
private String playerName;
public PlayerTarget(int playerId, String playerName)
{
this.playerId = playerId;
this.playerName = playerName;
}
public int getPlayerId()
{
return playerId;
}
public String getPlayerName()
{
return playerName;
}
@Override
public String toString()
{
return String.format("Player [ %d, %s ]", this.playerId, this.playerName);
}
}

View File

@ -0,0 +1,45 @@
package data.target;
import data.TargetDataset;
public class QuestTarget implements TargetDataset
{
private int questId;
private String name;
private String involvedCharacters;
private String dialogue;
public QuestTarget(int questId, String name, String involvedCharacters, String dialogue)
{
this.questId = questId;
this.name = name;
this.involvedCharacters = involvedCharacters;
this.dialogue = dialogue;
}
public int getQuestId()
{
return questId;
}
public String getName()
{
return name;
}
public String getInvolvedCharacters()
{
return involvedCharacters;
}
public String getDialogue()
{
return dialogue;
}
@Override
public String toString()
{
return String.format("Quest [ %d, %s, %s, %s... ]", this.questId, this.name, this.involvedCharacters, this.dialogue.substring(0, 10));
}
}

View File

@ -0,0 +1,39 @@
package data.target;
import data.TargetDataset;
public class RelationshipTarget implements TargetDataset
{
private int playerId;
private int personId;
private int relationshipLevel;
public RelationshipTarget(int playerId, int personId, int relationshipLevel)
{
this.playerId = playerId;
this.personId = personId;
this.relationshipLevel = relationshipLevel;
}
public int getPlayerId()
{
return playerId;
}
public int getPersonId()
{
return personId;
}
public int getRelationshipLevel()
{
return relationshipLevel;
}
@Override
public String toString()
{
return String.format("Relationship [ %d, %d, %d ]", this.playerId, this.personId, this.relationshipLevel);
}
}

View File

@ -1,26 +1,26 @@
package etl; package etl;
import data.Dataset; import data.SourceDataset;
import utils.DataStorer; import utils.DataStorer;
import utils.StatementPreparer; import utils.StatementPreparerExtractor;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
public class Extractor<T extends Dataset> public class Extractor<T extends SourceDataset>
{ {
private Connection sourceDatabase; private Connection sourceDatabase;
private DataStorer<T> dataStorer; private DataStorer<T> dataStorer;
private ResultSet resultSet; private StatementPreparerExtractor statementPreparer;
private String sql; private String sql;
public Extractor(Connection sourceDatabase, DataStorer<T> dataStorer, public Extractor(Connection sourceDatabase, DataStorer<T> dataStorer,
String sql) StatementPreparerExtractor statementPreparer, String sql)
{ {
this.sourceDatabase = sourceDatabase; this.sourceDatabase = sourceDatabase;
this.dataStorer = dataStorer; this.dataStorer = dataStorer;
this.statementPreparer = statementPreparer;
this.sql = sql; this.sql = sql;
} }
@ -29,9 +29,10 @@ public class Extractor<T extends Dataset>
try try
{ {
var preparedStatement = this.sourceDatabase.prepareStatement(sql); var preparedStatement = this.sourceDatabase.prepareStatement(sql);
this.resultSet = preparedStatement.executeQuery(); this.statementPreparer.doPrepare(preparedStatement);
var resultSet = preparedStatement.executeQuery();
this.sourceDatabase.close(); this.sourceDatabase.close();
return this.dataStorer.doStore(this.resultSet); return this.dataStorer.doStore(resultSet);
} }
catch (SQLException e) catch (SQLException e)
{ {

View File

@ -1,24 +1,25 @@
package etl; package etl;
import data.Dataset; import data.SourceDataset;
import utils.StatementPreparer; import data.TargetDataset;
import utils.StatementPreparerLoader;
import java.sql.Connection; import java.sql.Connection;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
public class Loader<T extends Dataset> public class Loader<T extends TargetDataset>
{ {
private Connection targetDatabase; private Connection targetDatabase;
private StatementPreparer<T> statementPreparer; private StatementPreparerLoader<T> statementPreparerLoader;
private List<T> transformedData; private List<T> transformedData;
private String sql; private String sql;
public Loader(Connection targetDatabase, StatementPreparer<T> statementPreparer, List<T> transformedData, public Loader(Connection targetDatabase, StatementPreparerLoader<T> statementPreparerLoader, List<T> transformedData,
String sql) String sql)
{ {
this.targetDatabase = targetDatabase; this.targetDatabase = targetDatabase;
this.statementPreparer = statementPreparer; this.statementPreparerLoader = statementPreparerLoader;
this.transformedData = transformedData; this.transformedData = transformedData;
this.sql = sql; this.sql = sql;
} }
@ -30,7 +31,7 @@ public class Loader<T extends Dataset>
for (T transformedDatum : this.transformedData) for (T transformedDatum : this.transformedData)
{ {
var preparedStatement = this.targetDatabase.prepareStatement(this.sql); var preparedStatement = this.targetDatabase.prepareStatement(this.sql);
this.statementPreparer.doPrepare(preparedStatement, transformedDatum); this.statementPreparerLoader.doPrepare(preparedStatement, transformedDatum);
preparedStatement.executeUpdate(); preparedStatement.executeUpdate();
} }

View File

@ -1,27 +1,28 @@
package etl; package etl;
import data.Dataset; import data.SourceDataset;
import data.TargetDataset;
import utils.DataTransformer; import utils.DataTransformer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class Transformer<T extends Dataset, E extends Dataset> public class Transformer<T extends SourceDataset, E extends TargetDataset>
{ {
private DataTransformer<T, E> transformer; private DataTransformer<T, E> transformer;
private List<T> datasets; private List<T> extractedData;
public Transformer(DataTransformer<T, E> transformer, List<T> datasets) public Transformer(DataTransformer<T, E> transformer, List<T> extractedData)
{ {
this.transformer = transformer; this.transformer = transformer;
this.datasets = datasets; this.extractedData = extractedData;
} }
public List<E> doTransform() public List<E> doTransform()
{ {
var transformed = new ArrayList<E>(); var transformed = new ArrayList<E>();
for (T dataset : this.datasets) for (T dataset : this.extractedData)
{ {
transformed.add(this.transformer.transform(dataset)); transformed.add(this.transformer.transform(dataset));
} }

View File

@ -1,12 +1,12 @@
package utils; package utils;
import data.Dataset; import data.SourceDataset;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
public interface DataStorer<T extends Dataset> public interface DataStorer<T extends SourceDataset>
{ {
List<T> doStore(ResultSet resultSet) throws SQLException; List<T> doStore(ResultSet resultSet) throws SQLException;
} }

View File

@ -1,10 +1,9 @@
package utils; package utils;
import data.Dataset; import data.SourceDataset;
import data.TargetDataset;
import java.util.List; public interface DataTransformer<T extends SourceDataset, E extends TargetDataset>
public interface DataTransformer<T extends Dataset, E extends Dataset>
{ {
E transform(T dataset); E transform(T dataset);
} }

View File

@ -0,0 +1,9 @@
package utils;
import java.sql.PreparedStatement;
import java.sql.SQLException;
public interface StatementPreparerExtractor
{
void doPrepare(PreparedStatement preparedStatement) throws SQLException;
}

View File

@ -1,11 +1,12 @@
package utils; package utils;
import data.Dataset; import data.SourceDataset;
import data.TargetDataset;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException; import java.sql.SQLException;
public interface StatementPreparer<T extends Dataset> public interface StatementPreparerLoader<T extends TargetDataset>
{ {
void doPrepare(PreparedStatement preparedStatement, T data) throws SQLException; void doPrepare(PreparedStatement preparedStatement, T data) throws SQLException;
} }