1.使用datax同步数据
2.模板json(已配置hdfs ha)：
{
"job": {
"content": [
{
"reader": {
"name": "postgresqlreader",
"parameter": {
"username": "hs_sync",
"password": "Pass2025",
"column": [
"order_date",
"day",
"iso_day_of_week",
"weekday_cn",
"weekday_en",
"weekday_short",
"is_weekend",
"iso_week",
"month",
"month_cn",
"month_en_full",
"month_en_short",
"quarter",
"year"
],
"connection": [
{
"table": [
"dim_calendar"
],
"jdbcUrl": [
"jdbc:postgresql://100.64.0.10:25432/hs_sync_data"
]
}
],
"fetchSize": 1000
}
},
"writer": {
"name": "hdfswriter",
"parameter": {
"defaultFS": "hdfs://nameservice1",
"hadoopConfig": {
"dfs.nameservices": "nameservice1",
"dfs.ha.namenodes.nameservice1": "nn1,nn2",
"dfs.namenode.rpc-address.nameservice1.nn1": "192.168.33.61:8020",
"dfs.namenode.rpc-address.nameservice1.nn2": "192.168.33.62:8020",
"dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
},
"fileType": "orc",
"path": "/tmp/data/test/",
"fileName": "dim_calendar",
"writeMode": "truncate",
"column": [
{
"name": "order_date",
"type": "date"
},
{
"name": "day",
"type": "smallint"
},
{
"name": "iso_day_of_week",
"type": "smallint"
},
{
"name": "weekday_cn",
"type": "string"
},
{
"name": "weekday_en",
"type": "string"
},
{
"name": "weekday_short",
"type": "string"
},
{
"name": "is_weekend",
"type": "boolean"
},
{
"name": "iso_week",
"type": "smallint"
},
{
"name": "month",
"type": "smallint"
},
{
"name": "month_cn",
"type": "string"
},
{
"name": "month_en_full",
"type": "string"
},
{
"name": "month_en_short",
"type": "string"
},
{
"name": "quarter",
"type": "string"
},
{
"name": "year",
"type": "smallint"
}
],
"fieldDelimiter": "\t",
"maxFileSize": 134217728,
"encoding": "UTF-8"
}
}
}
],
"setting": {
"speed": {
"channel": 5
},
"errorLimit": {
"record": 0,
"percentage": 0.02
},
"retry": {
"limit": 3,
"interval": 5000
}
}
}
}