Spark RDD - Task not serializable记录

开始编写的程序:

public class SortTest {
    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("SortTest");
        JavaSparkContext sc = new JavaSparkContext(conf);
        sc.setLogLevel("WARN");

        JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(5, 4, 2, 9, 18, 1));

        JavaRDD<Integer> integerJavaRDD = rdd.sortBy(x -> x, true, 1);
        System.out.println(StringUtils.join(integerJavaRDD.collect(), ","));

        JavaPairRDD<String, Integer> rdd2 = sc.parallelizePairs(Arrays.asList(new Tuple2<>("b", 1), new Tuple2<>("c", 1), new Tuple2<>("a", 1)));
        rdd2 = rdd2.sortByKey(new Comparator<String>() {
            @Override
            public int compare(String o1, String o2) {
                return o1.compareTo(o2);
            }
        });
        System.out.println(StringUtils.join(rdd2.collect(), ","));

    }
}

运行时出现：

 

任务没有序列化。根据所学，main函数所在的即为Driver端的代码，这里提示SortTest没有序列化。但是SortTest类为主类，并不需要序列化。另外，算子里面的代码是在Executor中运行，则这里的sortByKey的参数，匿名比较器则是在Executor中运行，则Comparator需要序列化。但是没有找到Java匿名类序列化的方法，则单独实现一个具名类实现Serializable接口来代替，如下：

public class SortTest {

    static class IntegerComparator implements Comparator<String>, Serializable {

        @Override
        public int compare(String o1, String o2) {
            return o1.compareTo(o2);
        }
    }

    public static void main(String[] args) {

        SparkConf conf = new SparkConf().setMaster("local[*]").setAppName("SortTest");
        JavaSparkContext sc = new JavaSparkContext(conf);
        sc.setLogLevel("WARN");

        JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(5, 4, 2, 9, 18, 1));

        JavaRDD<Integer> integerJavaRDD = rdd.sortBy(x -> x, true, 1);
        System.out.println(StringUtils.join(integerJavaRDD.collect(), ","));

        JavaPairRDD<String, Integer> rdd2 = sc.parallelizePairs(Arrays.asList(new Tuple2<>("b", 1), new Tuple2<>("c", 1), new Tuple2<>("a", 1)));
        rdd2 = rdd2.sortByKey(new IntegerComparator());
        System.out.println(StringUtils.join(rdd2.collect(), ","));

        sc.stop();
    }
}