A grande migração contínua: trabalhos do CDC com o Confluent Cloud e o Relational Migrator

1
CREATE TABLE world_happiness_report (
2
  overall_rank int DEFAULT NULL,
3
  country_or_region text,
4
  score double DEFAULT NULL,
5
  gdp_per_capita double DEFAULT NULL,
6
  social_support double DEFAULT NULL,
7
  healthy_life_expectancy double DEFAULT NULL,
8
  freedom_to_make_life_choices double DEFAULT NULL,
9
  generosity double DEFAULT NULL,
10
  perceptions_of_corruption double DEFAULT NULL
11
);
12
13
14
LOAD DATA LOCAL INFILE 'C:/Your/Path/2019.csv'
15
INTO TABLE world_happiness_report
16
FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n'
17
IGNORE 1 ROWS;

1
cd ~/Library/Application\ Support /MongoDB/Relational\ Migrator/
2
ls 

1
nano user.properties

1
spring.profiles.active: confluent 

1
migrator.confluent.environment.environment-id: <Environment ID>
2
migrator.confluent.environment.cluster-id: <Cluster ID>
3
migrator.confluent.environment.bootstrap-server: <xx.region.provider.confluent.cloud:9092>
4
migrator.confluent.environment.rest-endpoint: <https://xxx.region.provider.confluent.cloud:443>
5
6
migrator.confluent.cloud-credentials.api-key: <Cloud API key>
7
migrator.confluent.cloud-credentials.api-secret: <Cloud API Secret>
8
9
migrator.confluent.kafka-credentials.api-key: <Cluster API key>
10
migrator.confluent.kafka-credentials.api-secret: <Cluster API Secret>

1
cd "/Applications/MongoDB Relational Migrator.app/Contents/app"
2
java -jar application-1.5.0.jar

1
/*
2
* Relational Migrator needs source database to allow change data capture.
3
* The following scripts must be executed on MySQL source database before starting migration.
4
* For more details, please see https://debezium.io/documentation/reference/stable/connectors/mysql.html#setting-up-mysql
5
*/
6
7
/*
8
* Before initiating migration job, the MySQL user is required to be able to connect to the source database.
9
* This MySQL user must have appropriate permissions on all databases for which the Relational Migrator is supposed to capture changes.
10
*
11
* Connect to Amazon RDS Mysql instance, follow the below link for instructions:
12
* https://dev.mysql.com/doc/mysql-cluster-excerpt/8.0/en/mysql-cluster-replication-schema.html
13
*
14
* Grant the required permissions to the user
15
*/
16
17
GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.*  TO 'anaiya'@'%' ;
18
19
/* Finalize the user’s permissions: */
20
FLUSH PRIVILEGES;
21
22
23
/* Furthermore, binary logging must be enabled for MySQL replication on AWS RDS instance. Please see the below for instructions:
24
* https://aws.amazon.com/premiumsupport/knowledge-center/enable-binary-logging-aurora/
25
*
26
* If the instance is using the default parameter group, you will need to create a new one before you can make any changes.
27
* For MySQL RDS instances, create a Parameter Group for your chosen MySQL version.
28
* For Aurora MySQL clusters, create a DB Cluster Parameter Group for your chosen MySQL version.
29
* Edit the group and set the "binlog_format" parameter to "ROW".
30
* Make sure your database or cluster is configured to use the new Parameter Group.
31
*
32
* Please note that you must reboot the database cluster or instance to apply changes, follow below for instructions:
33
* https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_RebootCluster.html
34
*/

1
UPDATE world_happiness_report
2
SET Score = 5.800
3
WHERE `country_or_region`= `Finland`
4
LIMIT 1;