java实现spark常用算子之cogroup

import org.apache.spark.SparkConf;import org.apache.spark.api.java.JavaPairRDD;import org.apache.spark.api.java.JavaSparkContext;import org.apache.spark.api.java.function.VoidFunction;import scala.Tuple2;import java.util.Arrays;import java.util.List;/** *  cogroup(otherDataSet,[numTasks]) 算子： *  将两个RDD中的数据按照key进行汇总： *      第一个RDD按照key进行汇总，结果放在iterable中。 *      第二个RDD同样按照key进行汇总，结果放在另一个iterable中。 *      最后得到一个key和两个iterable的数据。 */public class CogroupOperator {    public static void main(String[] args) {        SparkConf conf = new SparkConf().setMaster("local").setAppName("cogroup");        JavaSparkContext sc = new JavaSparkContext(conf);        List<Tuple2<String,String>> stus = Arrays.asList(                new Tuple2<>("w1","z1"),                new Tuple2<>("w2","z2"),                new Tuple2<>("w3","z3"),                new Tuple2<>("w2","z2"),                new Tuple2<>("w1","z1")        );        List<Tuple2<String,String>> scores = Arrays.asList(                new Tuple2<>("w1","100"),                new Tuple2<>("w2","10"),                new Tuple2<>("w3","1"),                new Tuple2<>("w2","90"),                new Tuple2<>("w1","900")        );        JavaPairRDD<String,String> stusrdd = sc.parallelizePairs(stus);        JavaPairRDD<String,String> scorerdd = sc.parallelizePairs(scores);        JavaPairRDD<String,Tuple2<Iterable<String>,Iterable<String>>> result = stusrdd.cogroup(scorerdd,2);        result.foreach(new VoidFunction<Tuple2<String, Tuple2<Iterable<String>, Iterable<String>>>>() {            @Override            public void call(Tuple2<String, Tuple2<Iterable<String>, Iterable<String>>> tuple) throws Exception {                System.err.println("key:"+tuple._1+",另外两个iterable中的数据:"+tuple._2);            }        });    }}

微信扫描下图二维码加入博主知识星球,获取更多大数据、人工智能、算法等免费学习资料哦！