记录一次spark连接mysql遇到的问题

  1. 云栖社区>
  2. 博客>
  3. 正文

记录一次spark连接mysql遇到的问题

终日而思一 2018-08-15 22:01:00 浏览707
展开阅读全文

在使用spark连接mysql的过程中报错了,错误如下

08:51:32.495 [main] ERROR  - Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factory
java.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider
        at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
        at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
        at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
        at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
        at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
        at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
        at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
        at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
        at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
        at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
        at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
        at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
        at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
        at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
        at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
        at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
        at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
        at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
        at java.lang.Class.forName0(Native Method) ~[?:1.8.0_144]
        at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_144]
        at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
        at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
        at org.apache.calcite.avatica.UnregisteredDriver.<init>(UnregisteredDriver.java:55) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
        at org.apache.calcite.jdbc.Driver.<init>(Driver.java:53) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating]
        at org.apache.calcite.jdbc.Driver.<clinit>(Driver.java:49) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating]
        at java.lang.Class.forName0(Native Method) [?:1.8.0_144]
        at java.lang.Class.forName(Class.java:348) [?:1.8.0_144]
        at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556) [?:1.8.0_144]
        at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548) [?:1.8.0_144]
        at java.sql.DriverManager.getDrivers(DriverManager.java:446) [?:1.8.0_144]
        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation.<init>(JDBCRelation.scala:113) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125) [spark-sql_2.11-2.1.0.jar:2.1.0]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:45) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:50) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:52) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:54) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:56) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:58) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:60) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:62) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw$$iw.<init>(<console>:64) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw$$iw.<init>(<console>:66) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw$$iw.<init>(<console>:68) [scala-library-2.11.8.jar:?]
        at $line31.$read$$iw.<init>(<console>:70) [scala-library-2.11.8.jar:?]
        at $line31.$read.<init>(<console>:72) [scala-library-2.11.8.jar:?]
        at $line31.$read$.<init>(<console>:76) [scala-library-2.11.8.jar:?]
        at $line31.$read$.<clinit>(<console>) [scala-library-2.11.8.jar:?]
        at $line31.$eval$.$print$lzycompute(<console>:7) [scala-library-2.11.8.jar:?]
        at $line31.$eval$.$print(<console>:6) [scala-library-2.11.8.jar:?]
        at $line31.$eval.$print(<console>) [scala-library-2.11.8.jar:?]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144]
        at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) [scala-compiler-2.11.8.jar:?]
        at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) [scala-reflect-2.11.8.jar:?]
        at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) [scala-reflect-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
        at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) [scala-reflect-2.11.8.jar:?]
        at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
        at org.apache.spark.repl.Main$.doMain(Main.scala:68) [spark-repl_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.repl.Main$.main(Main.scala:51) [spark-repl_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.repl.Main.main(Main.scala) [spark-repl_2.11-2.1.0.jar:2.1.0]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144]
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) [spark-core_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) [spark-core_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) [spark-core_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) [spark-core_2.11-2.1.0.jar:2.1.0]
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) [spark-core_2.11-2.1.0.jar:2.1.0]
Caused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider
        at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
        at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
        ... 110 more
java.lang.RuntimeException: Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factory
  at org.apache.calcite.avatica.UnregisteredDriver.handle(UnregisteredDriver.java:132)
  at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:126)
  at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74)
  at org.apache.calcite.avatica.UnregisteredDriver.<init>(UnregisteredDriver.java:55)
  at org.apache.calcite.jdbc.Driver.<init>(Driver.java:53)
  at org.apache.calcite.jdbc.Driver.<clinit>(Driver.java:49)
  at java.lang.Class.forName0(Native Method)
  at java.lang.Class.forName(Class.java:348)
  at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556)
  at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548)
  at java.sql.DriverManager.getDrivers(DriverManager.java:446)
  at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52)
  at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50)
  at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58)
  at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation.<init>(JDBCRelation.scala:113)
  at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45)
  at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330)
  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152)
  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125)
  ... 54 elided
Caused by: java.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider
  at java.lang.ClassLoader.defineClass1(Native Method)
  at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
  at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
  at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
  at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
  at java.security.AccessController.doPrivileged(Native Method)
  at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
  at java.lang.ClassLoader.defineClass1(Native Method)
  at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
  at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
  at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
  at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
  at java.security.AccessController.doPrivileged(Native Method)
  at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
  at java.lang.ClassLoader.defineClass1(Native Method)
  at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
  at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
  at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
  at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
  at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
  at java.security.AccessController.doPrivileged(Native Method)
  at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
  at java.lang.Class.forName0(Native Method)
  at java.lang.Class.forName(Class.java:264)
  at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115)
  ... 71 more
Caused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider
  at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
  at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
  ... 110 more  

连接Mysql的代码是没问题的,但是居然会报这个错误,感觉很奇怪,而且去网上搜还搜不到相关资料,难道只有我碰到这个错误?

 

一开始以为是mysql的连接jar包有问题,但下载到本地用java连接又可以连接得上Mysql。

 

这时候注意到了错误ClassNotFoundException,一般这个错误要么是包冲突,要么是包缺失,包冲突就比较恶心了,所以先排查看看会不会是包缺失的问题,在Idea中键入两个shift,在搜素框中搜素org.apache.calcite.jdbc.CalciteJdbc41Factory。找到了这个东西,再去依赖包中查看相关jar包,发现calcite相关的有三个jar包,而在spark的jar包路径下calcite相关jar包只有两个,OK,在mven上下载缺失jar包,问题解决。

 

 


 


网友评论

登录后评论
0/500
评论