From 6b23c32c6d8a61adf2106dc31802fa47af405e63 Mon Sep 17 00:00:00 2001 From: Niklas Birk Date: Wed, 5 Feb 2020 18:41:37 +0100 Subject: [PATCH] initial --- pom.xml | 29 ++++++++ src/main/java/Main.java | 75 ++++++++++++++++++++ src/main/java/data/Dataset.java | 5 ++ src/main/java/data/source/PersonSource.java | 38 ++++++++++ src/main/java/data/target/PersonTarget.java | 38 ++++++++++ src/main/java/etl/Extractor.java | 43 +++++++++++ src/main/java/etl/Loader.java | 44 ++++++++++++ src/main/java/etl/Transformer.java | 30 ++++++++ src/main/java/utils/ConnectionHelper.java | 65 +++++++++++++++++ src/main/java/utils/DataStorer.java | 12 ++++ src/main/java/utils/DataTransformer.java | 10 +++ src/main/java/utils/DatabaseHelper.java | 5 ++ src/main/java/utils/DatabaseInformation.java | 51 +++++++++++++ src/main/java/utils/DatabaseType.java | 27 +++++++ src/main/java/utils/StatementPreparer.java | 11 +++ 15 files changed, 483 insertions(+) create mode 100644 pom.xml create mode 100644 src/main/java/Main.java create mode 100644 src/main/java/data/Dataset.java create mode 100644 src/main/java/data/source/PersonSource.java create mode 100644 src/main/java/data/target/PersonTarget.java create mode 100644 src/main/java/etl/Extractor.java create mode 100644 src/main/java/etl/Loader.java create mode 100644 src/main/java/etl/Transformer.java create mode 100644 src/main/java/utils/ConnectionHelper.java create mode 100644 src/main/java/utils/DataStorer.java create mode 100644 src/main/java/utils/DataTransformer.java create mode 100644 src/main/java/utils/DatabaseHelper.java create mode 100644 src/main/java/utils/DatabaseInformation.java create mode 100644 src/main/java/utils/DatabaseType.java create mode 100644 src/main/java/utils/StatementPreparer.java diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..201561d --- /dev/null +++ b/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + + org.example + DataMigration_Selfmade + 1.0-SNAPSHOT + + + + mysql + mysql-connector-java + 8.0.19 + + + org.mariadb.jdbc + mariadb-java-client + 2.5.4 + + + org.postgresql + postgresql + 42.2.9 + + + + \ No newline at end of file diff --git a/src/main/java/Main.java b/src/main/java/Main.java new file mode 100644 index 0000000..93ac12b --- /dev/null +++ b/src/main/java/Main.java @@ -0,0 +1,75 @@ +import data.source.PersonSource; +import data.target.PersonTarget; +import etl.Extractor; +import etl.Loader; +import etl.Transformer; +import utils.*; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Period; +import java.util.ArrayList; +import java.util.List; + +public class Main +{ + public static void main(final String[] args) throws SQLException + { +// mysqlDb(); +// postgresqlDb(); +// mariaDb(); + + var p = testExtract(); + var pt = testTransform(p); + testLoad(pt); + } + + private static List testExtract() throws SQLException + { + final var dbInfo = new DatabaseInformation("localhost", "sourcedb1", "test", "test", 5435); + final var connection = new ConnectionHelper(DatabaseType.MARIADB, dbInfo).createConnection(); + + DataStorer personSourceDataStorer = (rs) -> { + var persons = new ArrayList(); + while (rs.next()) + { + persons.add(new PersonSource(rs.getInt("personId"), + rs.getString("name"), + rs.getBoolean("mortal"))); + } + return persons; + }; + + var sql = "select * from person;"; + + return new Extractor<>(connection, personSourceDataStorer, sql).doExtract(); + } + + private static List testTransform(List persons) + { + DataTransformer personTransformer = (personSource) -> { + return new PersonTarget(personSource.getPersonId(), + personSource.getName(), + personSource.isMortal()); + }; + + return new Transformer<>(personTransformer, persons).doTransform(); + } + + private static void testLoad(List transformedData) + { + final var dbInfo = new DatabaseInformation("localhost", "targetdb", "test", "test", 5432); + final var connection = new ConnectionHelper(DatabaseType.POSTGRESQL, dbInfo).createConnection(); + + StatementPreparer statementPreparer = (preparedStatement, data) -> { + preparedStatement.setInt(1, data.getPersonId()); + preparedStatement.setString(2, data.getName()); + preparedStatement.setBoolean(3, data.isMortal()); + }; + + var sql = "insert into person values (?, ?, ?)"; + + new Loader<>(connection, statementPreparer, transformedData, sql).doLoad(); + } +} diff --git a/src/main/java/data/Dataset.java b/src/main/java/data/Dataset.java new file mode 100644 index 0000000..1cf08a1 --- /dev/null +++ b/src/main/java/data/Dataset.java @@ -0,0 +1,5 @@ +package data; + +public interface Dataset +{ +} diff --git a/src/main/java/data/source/PersonSource.java b/src/main/java/data/source/PersonSource.java new file mode 100644 index 0000000..4dafade --- /dev/null +++ b/src/main/java/data/source/PersonSource.java @@ -0,0 +1,38 @@ +package data.source; + +import data.Dataset; + +public class PersonSource implements Dataset +{ + private int personId; + private String name; + private boolean mortal; + + public PersonSource(int personId, String name, boolean mortal) + { + this.personId = personId; + this.name = name; + this.mortal = mortal; + } + + public int getPersonId() + { + return personId; + } + + public String getName() + { + return name; + } + + public boolean isMortal() + { + return mortal; + } + + @Override + public String toString() + { + return String.format("Person { %d, %s, %s }", this.personId, this.name, this.mortal); + } +} diff --git a/src/main/java/data/target/PersonTarget.java b/src/main/java/data/target/PersonTarget.java new file mode 100644 index 0000000..2f5754d --- /dev/null +++ b/src/main/java/data/target/PersonTarget.java @@ -0,0 +1,38 @@ +package data.target; + +import data.Dataset; + +public class PersonTarget implements Dataset +{ + private int personId; + private String name; + private boolean mortal; + + public PersonTarget(int personId, String name, boolean mortal) + { + this.personId = personId; + this.name = name; + this.mortal = mortal; + } + + public int getPersonId() + { + return personId; + } + + public String getName() + { + return name; + } + + public boolean isMortal() + { + return mortal; + } + + @Override + public String toString() + { + return String.format("Person [ %d, %s, %s ]", this.personId, this.name, this.mortal); + } +} diff --git a/src/main/java/etl/Extractor.java b/src/main/java/etl/Extractor.java new file mode 100644 index 0000000..061a6e8 --- /dev/null +++ b/src/main/java/etl/Extractor.java @@ -0,0 +1,43 @@ +package etl; + +import data.Dataset; +import utils.DataStorer; +import utils.StatementPreparer; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +public class Extractor +{ + private Connection sourceDatabase; + private DataStorer dataStorer; + private ResultSet resultSet; + private String sql; + + public Extractor(Connection sourceDatabase, DataStorer dataStorer, + String sql) + { + this.sourceDatabase = sourceDatabase; + this.dataStorer = dataStorer; + this.sql = sql; + } + + public List doExtract() + { + try + { + var preparedStatement = this.sourceDatabase.prepareStatement(sql); + this.resultSet = preparedStatement.executeQuery(); + this.sourceDatabase.close(); + return this.dataStorer.doStore(this.resultSet); + } + catch (SQLException e) + { + e.printStackTrace(); + } + + return null; + } +} diff --git a/src/main/java/etl/Loader.java b/src/main/java/etl/Loader.java new file mode 100644 index 0000000..3809492 --- /dev/null +++ b/src/main/java/etl/Loader.java @@ -0,0 +1,44 @@ +package etl; + +import data.Dataset; +import utils.StatementPreparer; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; + +public class Loader +{ + private Connection targetDatabase; + private StatementPreparer statementPreparer; + private List transformedData; + private String sql; + + public Loader(Connection targetDatabase, StatementPreparer statementPreparer, List transformedData, + String sql) + { + this.targetDatabase = targetDatabase; + this.statementPreparer = statementPreparer; + this.transformedData = transformedData; + this.sql = sql; + } + + public void doLoad() + { + try + { + for (T transformedDatum : this.transformedData) + { + var preparedStatement = this.targetDatabase.prepareStatement(this.sql); + this.statementPreparer.doPrepare(preparedStatement, transformedDatum); + preparedStatement.executeUpdate(); + } + + this.targetDatabase.close(); + } + catch (SQLException e) + { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/etl/Transformer.java b/src/main/java/etl/Transformer.java new file mode 100644 index 0000000..78085b0 --- /dev/null +++ b/src/main/java/etl/Transformer.java @@ -0,0 +1,30 @@ +package etl; + +import data.Dataset; +import utils.DataTransformer; + +import java.util.ArrayList; +import java.util.List; + +public class Transformer +{ + + private DataTransformer transformer; + private List datasets; + + public Transformer(DataTransformer transformer, List datasets) + { + this.transformer = transformer; + this.datasets = datasets; + } + + public List doTransform() + { + var transformed = new ArrayList(); + for (T dataset : this.datasets) + { + transformed.add(this.transformer.transform(dataset)); + } + return transformed; + } +} diff --git a/src/main/java/utils/ConnectionHelper.java b/src/main/java/utils/ConnectionHelper.java new file mode 100644 index 0000000..61dd99c --- /dev/null +++ b/src/main/java/utils/ConnectionHelper.java @@ -0,0 +1,65 @@ +package utils; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.*; +import java.util.Properties; + +public class ConnectionHelper +{ + private final DatabaseType databaseType; + private final String user; + private final String password; + private URI uri; + + public ConnectionHelper(final DatabaseType databaseType, final DatabaseInformation databaseInformation) + { + this.databaseType = databaseType; + + try + { + this.uri = new URI(this.databaseType.getScheme(), + null, + databaseInformation.getHost(), + databaseInformation.getPort(), + String.format("/%s", databaseInformation.getDatabaseName()), + null, + null); + } + catch (final URISyntaxException e) + { + e.printStackTrace(); + } + + this.user = databaseInformation.getUser(); + this.password = databaseInformation.getPassword(); + } + + public Connection createConnection() + { + try + { + Class.forName(this.databaseType.getDriver()); + } + catch (final ClassNotFoundException e) + { + e.printStackTrace(); + } + + Connection c = null; + + try + { + c = DriverManager.getConnection(this.uri.toString(), this.user, this.password); + } + catch (final SQLException e) + { + System.out.println("SQLException: " + e.getMessage()); + System.out.println("SQLState: " + e.getSQLState()); + System.out.println("VendorError: " + e.getErrorCode()); + e.printStackTrace(); + } + + return c; + } +} diff --git a/src/main/java/utils/DataStorer.java b/src/main/java/utils/DataStorer.java new file mode 100644 index 0000000..84f02d3 --- /dev/null +++ b/src/main/java/utils/DataStorer.java @@ -0,0 +1,12 @@ +package utils; + +import data.Dataset; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +public interface DataStorer +{ + List doStore(ResultSet resultSet) throws SQLException; +} diff --git a/src/main/java/utils/DataTransformer.java b/src/main/java/utils/DataTransformer.java new file mode 100644 index 0000000..f934cfc --- /dev/null +++ b/src/main/java/utils/DataTransformer.java @@ -0,0 +1,10 @@ +package utils; + +import data.Dataset; + +import java.util.List; + +public interface DataTransformer +{ + E transform(T dataset); +} diff --git a/src/main/java/utils/DatabaseHelper.java b/src/main/java/utils/DatabaseHelper.java new file mode 100644 index 0000000..80debfa --- /dev/null +++ b/src/main/java/utils/DatabaseHelper.java @@ -0,0 +1,5 @@ +package utils; + +public class DatabaseHelper +{ +} diff --git a/src/main/java/utils/DatabaseInformation.java b/src/main/java/utils/DatabaseInformation.java new file mode 100644 index 0000000..0e0c088 --- /dev/null +++ b/src/main/java/utils/DatabaseInformation.java @@ -0,0 +1,51 @@ +package utils; + +import com.mysql.cj.exceptions.NumberOutOfRange; + +public class DatabaseInformation +{ + private final String host; + private final String databaseName; + private final String user; + private final String password; + + private int port; + + public DatabaseInformation(final String host, final String databaseName, final String user, final String password, final int port) + { + this.host = host; + this.databaseName = databaseName; + this.user = user; + this.password = password; + + if (port < 0 || port > 65535) + throw new NumberOutOfRange("A port must be between 0 and 65535"); + else + this.port = port; + } + + public String getHost() + { + return host; + } + + public String getDatabaseName() + { + return databaseName; + } + + public String getUser() + { + return user; + } + + public String getPassword() + { + return password; + } + + public int getPort() + { + return port; + } +} \ No newline at end of file diff --git a/src/main/java/utils/DatabaseType.java b/src/main/java/utils/DatabaseType.java new file mode 100644 index 0000000..0150064 --- /dev/null +++ b/src/main/java/utils/DatabaseType.java @@ -0,0 +1,27 @@ +package utils; + +public enum DatabaseType +{ + MARIADB("org.mariadb.jdbc.Driver", "jdbc:mariadb"), + MYSQL("com.mysql.cj.jdbc.Driver", "jdbc:mysql"), + POSTGRESQL("org.postgresql.Driver", "jdbc:postgresql"); + + private final String driver; + private final String scheme; + + DatabaseType(final String driver, final String scheme) + { + this.driver = driver; + this.scheme = scheme; + } + + public String getDriver() + { + return driver; + } + + public String getScheme() + { + return scheme; + } +} \ No newline at end of file diff --git a/src/main/java/utils/StatementPreparer.java b/src/main/java/utils/StatementPreparer.java new file mode 100644 index 0000000..9c2f373 --- /dev/null +++ b/src/main/java/utils/StatementPreparer.java @@ -0,0 +1,11 @@ +package utils; + +import data.Dataset; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +public interface StatementPreparer +{ + void doPrepare(PreparedStatement preparedStatement, T data) throws SQLException; +}