spark 3.5.8 编译
当执行insert 命令:INSERT INTO hudi_table_spark
VALUES
(1695091554788, 'e96c4396-3fad-413a-a942-4cb36106d721', 'rider-C', 'driver-M', -2.70, 'san_francisco'),
(1695530237068, '3f3d9565-7261-40e6-9b39-b8aa784f95e2', 'rider-K', 'driver-U', 64.20, 'san_francisco'),
(1695241330902, 'ea4c36ff-2069-4148-9927-ef8c1a5abd24', 'rider-H', 'driver-R', 66.60, 'sao_paulo'),
(1695115999911, 'c8abbe79-8d89-47ea-b4ce-4d224bae5bfa', 'rider-J', 'driver-T', 1.85, 'chennai');
报错信息:
. 33 more
Caused by: org.apache.hudi.exception.HoodieException: Unable to instantiate class org.apache.hudi.storage.hadoop.HoodieHadoopStorage
at org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:75)
at org.apache.hudi.storage.HoodieStorageUtils.getStorage(HoodieStorageUtils.java:41)
... 44 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:73)
... 45 more
Caused by: java.lang.NoClassDefFoundError: com/google/protobuf/RpcController
at java.lang.Class.getDeclaredMethods0(Native Method)
at java.lang.Class.privateGetDeclaredMethods(Class.java:2701)
at java.lang.Class.privateGetPublicMethods(Class.java:2902)
at java.lang.Class.privateGetPublicMethods(Class.java:2911)
at java.lang.Class.getMethods(Class.java:1615)
at sun.misc.ProxyGenerator.generateClassFile(ProxyGenerator.java:451)
at sun.misc.ProxyGenerator.generateProxyClass(ProxyGenerator.java:339)
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:639)
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:557)
at java.lang.reflect.WeakCache$Factory.get(WeakCache.java:230)
at java.lang.reflect.WeakCache.get(WeakCache.java:127)
at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:419)
at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:719)
at org.apache.hadoop.ipc.ProtobufRpcEngine.getProxy(ProtobufRpcEngine.java:101)
at org.apache.hadoop.ipc.RPC.getProtocolProxy(RPC.java:585)
at org.apache.hadoop.hdfs.NameNodeProxiesClient.createNonHAProxyWithClientProtocol(NameNodeProxiesClient.java:354)
at org.apache.hadoop.hdfs.server.namenode.ha.ClientHAProxyFactory.createProxy(ClientHAProxyFactory.java:34)
at org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.getProxy(ConfiguredFailoverProxyProvider.java:127)
at org.apache.hadoop.io.retry.RetryInvocationHandler$ProxyDescriptor.
at org.apache.hadoop.io.retry.RetryInvocationHandler.
at org.apache.hadoop.io.retry.RetryInvocationHandler.
at org.apache.hadoop.io.retry.RetryProxy.create(RetryProxy.java:59)
at org.apache.hadoop.hdfs.NameNodeProxiesClient.createHAProxy(NameNodeProxiesClient.java:319)
at org.apache.hadoop.hdfs.NameNodeProxiesClient.createProxyWithClientProtocol(NameNodeProxiesClient.java:137)
at org.apache.hadoop.hdfs.DFSClient.
at org.apache.hadoop.hdfs.DFSClient.
at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:167)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3288)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:473)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:361)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:125)
at org.apache.hudi.hadoop.fs.HadoopFSUtils.getFs(HadoopFSUtils.java:118)
at org.apache.hudi.storage.hadoop.HoodieHadoopStorage.
... 50 more
Caused by: java.lang.ClassNotFoundException: com.google.protobuf.RpcController
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 83 more
经过分析spark/jars目录存在此类:com.google.protobuf.RpcController
[root@management-87f97c38 jars]# pwd
/usr/hdp/3.0.1.0-187/spark-3.5.8/jars
You have mail in /var/spool/mail/root
[root@management-87f97c38 jars]#
[root@management-87f97c38 jars]#
[root@management-87f97c38 jars]# find . -name '*.jar' | while read jarfile; do if jar tf "$jarfile" | grep com/google/protobuf/RpcController -q; then echo "$jarfile"; fi; done
./hudi-spark3.5-bundle_2.12-1.1.1.jar
./hadoop-client-runtime-3.0.0-cdh6.2.0.jar
[root@management-87f97c38 jars]#
浙公网安备 33010602011771号