1package com.konka.dsp; 2 3 4import org.apache.flink.api.common.JobExecutionResult; 5import org.apache.flink.api.common.restartstrategy.RestartStrategies; 6import org.apache.flink.api.java.DataSet; 7import org.apache.flink.api.java.ExecutionEnvironment; 8import org.apache.flink.api.java.tuple.Tuple2; 9import org.apache.flink.connector.jdbc.dialect.MySQLDialect; 10import org.apache.flink.connector.jdbc.internal.options.JdbcOptions; 11import org.apache.flink.connector.jdbc.table.JdbcUpsertTableSink; 12import org.apache.flink.formats.json.JsonFormatFactory; 13import org.apache.flink.formats.json.canal.CanalJsonFormatFactory; 14import org.apache.flink.shaded.curator4.org.apache.curator.framework.schema.Schema; 15import org.apache.flink.streaming.api.CheckpointingMode; 16import org.apache.flink.streaming.api.TimeCharacteristic; 17import org.apache.flink.streaming.api.datastream.DataStream; 18import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; 19import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 20import org.apache.flink.table.api.*; 21import org.apache.flink.table.api.bridge.java.BatchTableEnvironment; 22import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 23import org.apache.flink.table.expressions.TimeIntervalUnit; 24import org.apache.flink.table.types.DataType; 25import org.apache.flink.types.Row; 26import org.apache.flink.util.CloseableIterator; 27import org.slf4j.Logger; 28import org.slf4j.LoggerFactory; 29 30 31import java.time.Duration; 32import java.util.concurrent.ExecutionException; 33 34import static org.apache.flink.table.api.Expressions.*; 35 36 37public class SalesOrderStream { 38 39 private static Logger log = LoggerFactory.getLogger(SalesOrderStream.class.getName()); 40 public static Table report(Table transactions) { 41 42 return transactions.select( 43 $("customer_name"), 44 $("created_date"), 45 $("total_amount")) 46 .groupBy($("customer_name"),$("created_date")) 47 .select( 48 $("customer_name"), 49 $("total_amount").sum().as("total_amount"), 50 $("created_date") 51 ); 52 53 } 54 55 public static void main(String[] args) throws Exception { 56 57 58 StreamExecutionEnvironment env = StreamExecutionEnvironment 59 .getExecutionEnvironment(); 60// env.setParallelism(4); 61// env.getConfig().setRestartStrategy(RestartStrategies.noRestart()); 62 StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); 63 64 // set default parallelism to 4 65 66 67// tEnv.executeSql("CREATE TABLE sales_order_header_stream (\n" + 68//// " id BIGINT not null,\n" + 69// " customer_name STRING,\n"+ 70//// " dsp_org_name STRING,\n"+ 71// " total_amount DECIMAL(38,2),\n" + 72//// " total_discount DECIMAL(16,2),\n" + 73//// " pay_amount DECIMAL(16,2),\n" + 74//// " total_amount DECIMAL(16,2),\n" + 75// " created_date TIMESTAMP(3)\n" + 76// ") WITH (\n" + 77// " 'connector' = 'mysql-cdc',\n" + 78// " 'hostname' = '192.168.8.73',\n" + 79// " 'port' = '4000',\n"+ 80// " 'username' = 'flink',\n"+ 81// " 'password' = 'flink',\n"+ 82// " 'database-name' = 'dspdev',\n"+ 83// " 'table-name' = 'sales_order_header'\n"+ 84// ")"); 85 //pay_type,over_sell 86 tEnv.executeSql("CREATE TABLE sales_order_header_stream (\n" + 87 " `id` BIGINT,\n"+ 88 " `total_amount` DECIMAL(16,2) ,\n"+ 89 " `customer_name` STRING,\n"+ 90 " `order_no` STRING,\n"+ 91 " `doc_type` STRING,\n"+ 92 " `sales_org` STRING,\n"+ 93 " `distr_chan` STRING,\n"+ 94 " `division` STRING,\n"+ 95 " `sales_grp` STRING,\n"+ 96 " `sales_off` STRING,\n"+ 97 " `purch_no_c` STRING,\n"+ 98 " `purch_date` STRING,\n"+ 99 " `sold_to` STRING,\n"+ 100 " `ship_to` STRING,\n"+ 101 " `r3_sales_order` STRING,\n"+ 102 " `created_by_employee_name` STRING,\n"+ 103 " `created_by_dept_name` STRING,\n"+ 104 " `created_by_dept_name` STRING,\n"+ 105 " `is_enable` BIGINT,\n"+ 106 " `is_delete` BIGINT,\n"+ 107 " `sale_order_status` STRING,\n"+ 108 " `created_by_parent_dept_name` STRING,\n"+ 109 " `total_discount` DECIMAL(16,2),\n"+ 110 " `customer_sapcode` STRING,\n"+ 111 " `sold_to_name` STRING,\n"+ 112 " `ship_to_name` STRING,\n"+ 113 " `total_discount_amount` DECIMAL(16,2),\n"+ 114 " `other_discount` DECIMAL(16,2),\n"+ 115 " `other_amount` DECIMAL(16,2),\n"+ 116 " `pay_amount` DECIMAL(16,2),\n"+ 117 " `dsp_org_name` STRING,\n"+ 118 " `delivery_address` STRING,\n"+ 119 " `delivery_person` STRING,\n"+ 120 " `delivery_phone` STRING,\n"+ 121 " `pay_type` STRING,\n"+ 122 " `over_sell` STRING,\n"+ 123 " `created_date` TIMESTAMP(3),\n"+ 124 " PRIMARY KEY (`id`) NOT ENFORCED "+ 125 ") WITH (\n" + 126 "'connector' = 'kafka',\n"+ 127 "'topic' = 'canal-data',\n"+ 128 "'properties.bootstrap.servers' = '192.168.8.71:9092',\n"+ 129 "'properties.group.id' = 'test',\n"+ 130 "'format' = 'canal-json'\n"+ 131 ")"); 132 133// tEnv.executeSql("CREATE TABLE total_day_report (\n" + 134// " customer_name STRING,\n" + 135//// " total_amount DECIMAL(16,2),\n" + 136//// " total_discount DECIMAL(16,2),\n" + 137//// " pay_amount DECIMAL(16,2),\n" + 138// " total_amount DECIMAL(16,2),\n" + 139// " created_date STRING,\n" + 140// " PRIMARY KEY (created_date) NOT ENFORCED" + 141// ") WITH (\n" + 142// " 'connector' = 'upsert-kafka',\n" + 143// " 'topic' = 'customer_amount',\n" + 144// " 'properties.bootstrap.servers' = '192.168.8.71:9092',\n"+ 145// " 'key.format' = 'json',\n"+ 146// " 'value.format' = 'json',\n"+ 147// " 'value.fields-include' = 'ALL'\n"+ 148// ")"); 149 150 151 152 tEnv.executeSql("CREATE TABLE upsertSink (\n" + 153 " customer_name STRING,\n" + 154// " total_amount DECIMAL(16,2),\n" + 155// " total_discount DECIMAL(16,2),\n" + 156// " pay_amount DECIMAL(16,2),\n" + 157 " total_amount DECIMAL(16,2),\n" + 158 " created_date STRING,\n" + 159 " PRIMARY KEY (customer_name,created_date) NOT ENFORCED" + 160 ") WITH (\n" + 161 " 'connector' = 'tidb',\n" + 162 " 'tidb.database.url' = 'jdbc:mysql://192.168.8.73:4000/dspdev',\n" + 163 " 'tidb.username' = 'flink',\n"+ 164 " 'tidb.password' = 'flink',\n"+ 165 " 'tidb.database.name' = 'dspdev',\n"+ 166 " 'tidb.table.name' = 'spend_report'\n"+ 167// " 'connector.type'='jdbc'," + 168// " 'connector.url'='jdbc:mysql://192.168.8.73:4000/dspdev',\n" + 169// " 'connector.username' = 'flink',\n"+ 170// " 'connector.password' = 'flink',\n"+ 171// " 'connector.table' = 'spend_report'" + 172 ")"); 173// TableSchema tableSche = TableSchema.builder() 174// .field("customer_name",DataTypes.STRING().notNull()) 175// .field("total_amount",DataTypes.DECIMAL(16,2)) 176// .field("created_date",DataTypes.STRING().notNull()).build(); 177// 178// JdbcOptions jdbcOptions = JdbcOptions.builder() 179// .setDBUrl("jdbc:mysql://192.168.8.73:4000/dspdev") 180// .setTableName("spend_report") 181// .setUsername("flink") 182// .setPassword("flink") 183// .setDialect(new MySQLDialect()) 184// .setDriverName("com.mysql.jdbc.Driver") 185// .build(); 186// JdbcUpsertTableSink jdbcUpsertTableSink = JdbcUpsertTableSink.builder() 187// .setTableSchema(tableSche) 188// .setOptions(jdbcOptions) 189// .build(); 190// jdbcUpsertTableSink.setKeyFields(new String[]{"id"}); 191 /** 192 * SINK End 193 */ 194// tEnv.re("spend_report",jdbcUpsertTableSink); 195 Table transactions = tEnv.from("sales_order_header_stream"); 196// tEnv.executeSql("delete from total_day_report"); 197 tEnv.executeSql("insert into upsertSink select dsp_org_name as customer_name,cast(sum(t.pay_amount) as decimal(16,2)) as amount,DATE_FORMAT(t.created_date,'yyyy-MM-dd') as created_date from sales_order_header_stream t group by DATE_FORMAT(t.created_date,'yyyy-MM-dd'),dsp_org_name").print(); 198// tEnv.executeSql("insert into spend_report select * from total_day_report"); 199// Table transactions = tEnv.from("total_day_report"); 200// report(transactions).executeInsert("spend_report"); 201 tEnv.execute("-----------"); 202 } 203 204 205} 206
最后数据库结果如下: 
每次都是更新替换，这样的话省去很多麻烦，不用转datastream在处理了，而且1.12支持upsert-kafka，最后数据叠加如下: 
upsert-kafka上面已经体现，flink越来越强大了支持！~~~