当我们尝试在 sbt 文件中使用 cloudera 上游版本时，sbt 程序集无法构建
sbt assembly failing to build when we try to use the cloudera upstream versions in the sbt file
我正在尝试使用 sbt-assembly 使用 Cloudera 上游版本构建一个 Scala 项目。它抛出以下异常：
>sbt assembly
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=256m; support was removed in 8.0
[info] Loading settings for project global-plugins from plugins.sbt ...
[info] Loading global plugins from C:\Users\avi\.sbt.0\plugins
[info] Updating ProjectRef(uri("file:/C:/Users/avi/.sbt/1.0/plugins/"), "global-plugins")...
[warn] module not found: com.typesafe.sbt#sbt-pgp;1.0-M1
[warn] ==== typesafe-ivy-releases: tried
[warn] https://repo.typesafe.com/typesafe/ivy-releases/com.typesafe.sbt/sbt-pgp/scala_2.12/sbt_1.0/1.0-M1/ivys/ivy.xml
[warn] ==== sbt-plugin-releases: tried
[warn] https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/com.typesafe.sbt/sbt-pgp/scala_2.12/sbt_1.0/1.0-M1/ivys/ivy.xml
[warn] ==== local: tried
[warn] C:\Users\avi\.ivy2\local\com.typesafe.sbt\sbt-pgp\scala_2.12\sbt_1.0.0-M1\ivys\ivy.xml
[warn] ==== public: tried
[warn] https://repo1.maven.org/maven2/com/typesafe/sbt/sbt-pgp_2.12_1.0/1.0-M1/sbt-pgp-1.0-M1.pom
[warn] ==== local-preloaded-ivy: tried
[warn] C:\Users\avi\.sbt\preloaded\com.typesafe.sbt\sbt-pgp.0-M1\ivys\ivy.xml
[warn] ==== local-preloaded: tried
[warn] file:/C:/Users/avi/.sbt/preloaded/com/typesafe/sbt/sbt-pgp_2.12_1.0/1.0-M1/sbt-pgp-1.0-M1.pom
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn] :: UNRESOLVED DEPENDENCIES ::
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn] :: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn]
[warn] Note: Some unresolved dependencies have extra attributes. Check that these dependencies exist with the requested attributes.
[warn] com.typesafe.sbt:sbt-pgp:1.0-M1 (scalaVersion=2.12, sbtVersion=1.0)
[warn]
[warn] Note: Unresolved dependencies path:
[warn] com.typesafe.sbt:sbt-pgp:1.0-M1 (scalaVersion=2.12, sbtVersion=1.0) (C:\Users\avi\.sbt.0\plugins\plugins.sbt#L3-4)
[warn] +- org.scala-sbt:global-plugins:0.0 (scalaVersion=2.12, sbtVersion=1.0)
Project loading failed: (r)etry, (q)uit, (l)ast, or (i)gnore? [error] sbt.librarymanagement.ResolveException: unresolved dependency: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
[error] at sbt.internal.librarymanagement.IvyActions$.resolveAndRetrieve(IvyActions.scala:332)
[error] at sbt.internal.librarymanagement.IvyActions$.$anonfun$updateEither(IvyActions.scala:208)
[error] at sbt.internal.librarymanagement.IvySbt$Module.$anonfun$withModule(Ivy.scala:239)
[error] at sbt.internal.librarymanagement.IvySbt.$anonfun$withIvy(Ivy.scala:204)
[error] at sbt.internal.librarymanagement.IvySbt.sbt$internal$librarymanagement$IvySbt$$action(Ivy.scala:70)
[error] at sbt.internal.librarymanagement.IvySbt$$anon.call(Ivy.scala:77)
[error] at xsbt.boot.Locks$GlobalLock.withChannel(Locks.scala:95)
[error] at xsbt.boot.Locks$GlobalLock.xsbt$boot$Locks$GlobalLock$$withChannelRetries(Locks.scala:80)
[error] at xsbt.boot.Locks$GlobalLock$$anonfun$withFileLock.apply(Locks.scala:99)
[error] at xsbt.boot.Using$.withResource(Using.scala:10)
[error] at xsbt.boot.Using$.apply(Using.scala:9)
[error] at xsbt.boot.Locks$GlobalLock.ignoringDeadlockAvoided(Locks.scala:60)
[error] at xsbt.boot.Locks$GlobalLock.withLock(Locks.scala:50)
[error] at xsbt.boot.Locks$.apply0(Locks.scala:31)
[error] at xsbt.boot.Locks$.apply(Locks.scala:28)
[error] at sbt.internal.librarymanagement.IvySbt.withDefaultLogger(Ivy.scala:77)
[error] at sbt.internal.librarymanagement.IvySbt.withIvy(Ivy.scala:199)
[error] at sbt.internal.librarymanagement.IvySbt.withIvy(Ivy.scala:196)
[error] at sbt.internal.librarymanagement.IvySbt$Module.withModule(Ivy.scala:238)
[error] at sbt.internal.librarymanagement.IvyActions$.updateEither(IvyActions.scala:193)
[error] at sbt.librarymanagement.ivy.IvyDependencyResolution.update(IvyDependencyResolution.scala:20)
[error] at sbt.librarymanagement.DependencyResolution.update(DependencyResolution.scala:56)
[error] at sbt.internal.LibraryManagement$.resolve(LibraryManagement.scala:45)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:93)
[error] at sbt.util.Tracked$.$anonfun$lastOutput(Tracked.scala:68)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:106)
[error] at scala.util.control.Exception$Catch.apply(Exception.scala:224)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:106)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate$adapted(LibraryManagement.scala:89)
[error] at sbt.util.Tracked$.$anonfun$inputChanged(Tracked.scala:149)
[error] at sbt.internal.LibraryManagement$.cachedUpdate(LibraryManagement.scala:120)
[error] at sbt.Classpaths$.$anonfun$updateTask(Defaults.scala:2561)
[error] at scala.Function1.$anonfun$compose(Function1.scala:44)
[error] at sbt.internal.util.$tilde$greater.$anonfun$$u2219(TypeFunctions.scala:40)
[error] at sbt.std.Transform$$anon.work(System.scala:67)
[error] at sbt.Execute.$anonfun$submit(Execute.scala:269)
[error] at sbt.internal.util.ErrorHandling$.wideConvert(ErrorHandling.scala:16)
[error] at sbt.Execute.work(Execute.scala:278)
[error] at sbt.Execute.$anonfun$submit(Execute.scala:269)
[error] at sbt.ConcurrentRestrictions$$anon.$anonfun$submitValid(ConcurrentRestrictions.scala:178)
[error] at sbt.CompletionService$$anon.call(CompletionService.scala:37)
[error] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[error] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[error] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[error] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[error] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[error] at java.lang.Thread.run(Thread.java:745)
[error] (update) sbt.librarymanagement.ResolveException: unresolved dependency: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
build.sbt
val sparkVersion = "2.3.0.cloudera4"
resolvers ++= Seq(
"cloudera" at "https://repository.cloudera.com/artifactory/cloudera-repos/"
)
libraryDependencies ++= Seq(
"org.apache.spark" % "spark-core_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-sql_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-hive_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-streaming_2.11" % sparkVersion % "provided"
)
还有 sbt 文件夹中的 plugins.sbt。
plugins.sbt :
addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
但是当我使用apache版本的时候。它构建成功。那么我应该添加任何其他依赖项以使 cloudera 上游版本正常工作吗？
看起来你已经设置了这个 sbt-plugin
com.typesafe.sbt:sbt-pgp:1.0-M1
在你的全局 sbt 配置中
C:\Users\avi\.sbt.0\plugins\plugins.sbt#L3-4
问题是版本 1.0-M1 很旧，与 sbt 1.0 和 scala 2.12
不兼容
您应该将插件升级到更新的版本：
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.1")
我正在尝试使用 sbt-assembly 使用 Cloudera 上游版本构建一个 Scala 项目。它抛出以下异常：
>sbt assembly
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option MaxPermSize=256m; support was removed in 8.0
[info] Loading settings for project global-plugins from plugins.sbt ...
[info] Loading global plugins from C:\Users\avi\.sbt.0\plugins
[info] Updating ProjectRef(uri("file:/C:/Users/avi/.sbt/1.0/plugins/"), "global-plugins")...
[warn] module not found: com.typesafe.sbt#sbt-pgp;1.0-M1
[warn] ==== typesafe-ivy-releases: tried
[warn] https://repo.typesafe.com/typesafe/ivy-releases/com.typesafe.sbt/sbt-pgp/scala_2.12/sbt_1.0/1.0-M1/ivys/ivy.xml
[warn] ==== sbt-plugin-releases: tried
[warn] https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/com.typesafe.sbt/sbt-pgp/scala_2.12/sbt_1.0/1.0-M1/ivys/ivy.xml
[warn] ==== local: tried
[warn] C:\Users\avi\.ivy2\local\com.typesafe.sbt\sbt-pgp\scala_2.12\sbt_1.0.0-M1\ivys\ivy.xml
[warn] ==== public: tried
[warn] https://repo1.maven.org/maven2/com/typesafe/sbt/sbt-pgp_2.12_1.0/1.0-M1/sbt-pgp-1.0-M1.pom
[warn] ==== local-preloaded-ivy: tried
[warn] C:\Users\avi\.sbt\preloaded\com.typesafe.sbt\sbt-pgp.0-M1\ivys\ivy.xml
[warn] ==== local-preloaded: tried
[warn] file:/C:/Users/avi/.sbt/preloaded/com/typesafe/sbt/sbt-pgp_2.12_1.0/1.0-M1/sbt-pgp-1.0-M1.pom
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn] :: UNRESOLVED DEPENDENCIES ::
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn] :: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
[warn] ::::::::::::::::::::::::::::::::::::::::::::::
[warn]
[warn] Note: Some unresolved dependencies have extra attributes. Check that these dependencies exist with the requested attributes.
[warn] com.typesafe.sbt:sbt-pgp:1.0-M1 (scalaVersion=2.12, sbtVersion=1.0)
[warn]
[warn] Note: Unresolved dependencies path:
[warn] com.typesafe.sbt:sbt-pgp:1.0-M1 (scalaVersion=2.12, sbtVersion=1.0) (C:\Users\avi\.sbt.0\plugins\plugins.sbt#L3-4)
[warn] +- org.scala-sbt:global-plugins:0.0 (scalaVersion=2.12, sbtVersion=1.0)
Project loading failed: (r)etry, (q)uit, (l)ast, or (i)gnore? [error] sbt.librarymanagement.ResolveException: unresolved dependency: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
[error] at sbt.internal.librarymanagement.IvyActions$.resolveAndRetrieve(IvyActions.scala:332)
[error] at sbt.internal.librarymanagement.IvyActions$.$anonfun$updateEither(IvyActions.scala:208)
[error] at sbt.internal.librarymanagement.IvySbt$Module.$anonfun$withModule(Ivy.scala:239)
[error] at sbt.internal.librarymanagement.IvySbt.$anonfun$withIvy(Ivy.scala:204)
[error] at sbt.internal.librarymanagement.IvySbt.sbt$internal$librarymanagement$IvySbt$$action(Ivy.scala:70)
[error] at sbt.internal.librarymanagement.IvySbt$$anon.call(Ivy.scala:77)
[error] at xsbt.boot.Locks$GlobalLock.withChannel(Locks.scala:95)
[error] at xsbt.boot.Locks$GlobalLock.xsbt$boot$Locks$GlobalLock$$withChannelRetries(Locks.scala:80)
[error] at xsbt.boot.Locks$GlobalLock$$anonfun$withFileLock.apply(Locks.scala:99)
[error] at xsbt.boot.Using$.withResource(Using.scala:10)
[error] at xsbt.boot.Using$.apply(Using.scala:9)
[error] at xsbt.boot.Locks$GlobalLock.ignoringDeadlockAvoided(Locks.scala:60)
[error] at xsbt.boot.Locks$GlobalLock.withLock(Locks.scala:50)
[error] at xsbt.boot.Locks$.apply0(Locks.scala:31)
[error] at xsbt.boot.Locks$.apply(Locks.scala:28)
[error] at sbt.internal.librarymanagement.IvySbt.withDefaultLogger(Ivy.scala:77)
[error] at sbt.internal.librarymanagement.IvySbt.withIvy(Ivy.scala:199)
[error] at sbt.internal.librarymanagement.IvySbt.withIvy(Ivy.scala:196)
[error] at sbt.internal.librarymanagement.IvySbt$Module.withModule(Ivy.scala:238)
[error] at sbt.internal.librarymanagement.IvyActions$.updateEither(IvyActions.scala:193)
[error] at sbt.librarymanagement.ivy.IvyDependencyResolution.update(IvyDependencyResolution.scala:20)
[error] at sbt.librarymanagement.DependencyResolution.update(DependencyResolution.scala:56)
[error] at sbt.internal.LibraryManagement$.resolve(LibraryManagement.scala:45)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:93)
[error] at sbt.util.Tracked$.$anonfun$lastOutput(Tracked.scala:68)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:106)
[error] at scala.util.control.Exception$Catch.apply(Exception.scala:224)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate(LibraryManagement.scala:106)
[error] at sbt.internal.LibraryManagement$.$anonfun$cachedUpdate$adapted(LibraryManagement.scala:89)
[error] at sbt.util.Tracked$.$anonfun$inputChanged(Tracked.scala:149)
[error] at sbt.internal.LibraryManagement$.cachedUpdate(LibraryManagement.scala:120)
[error] at sbt.Classpaths$.$anonfun$updateTask(Defaults.scala:2561)
[error] at scala.Function1.$anonfun$compose(Function1.scala:44)
[error] at sbt.internal.util.$tilde$greater.$anonfun$$u2219(TypeFunctions.scala:40)
[error] at sbt.std.Transform$$anon.work(System.scala:67)
[error] at sbt.Execute.$anonfun$submit(Execute.scala:269)
[error] at sbt.internal.util.ErrorHandling$.wideConvert(ErrorHandling.scala:16)
[error] at sbt.Execute.work(Execute.scala:278)
[error] at sbt.Execute.$anonfun$submit(Execute.scala:269)
[error] at sbt.ConcurrentRestrictions$$anon.$anonfun$submitValid(ConcurrentRestrictions.scala:178)
[error] at sbt.CompletionService$$anon.call(CompletionService.scala:37)
[error] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[error] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[error] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[error] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[error] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[error] at java.lang.Thread.run(Thread.java:745)
[error] (update) sbt.librarymanagement.ResolveException: unresolved dependency: com.typesafe.sbt#sbt-pgp;1.0-M1: not found
build.sbt
val sparkVersion = "2.3.0.cloudera4"
resolvers ++= Seq(
"cloudera" at "https://repository.cloudera.com/artifactory/cloudera-repos/"
)
libraryDependencies ++= Seq(
"org.apache.spark" % "spark-core_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-sql_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-hive_2.11" % sparkVersion % "provided",
"org.apache.spark" % "spark-streaming_2.11" % sparkVersion % "provided"
)
还有 sbt 文件夹中的 plugins.sbt。
plugins.sbt :
addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
但是当我使用apache版本的时候。它构建成功。那么我应该添加任何其他依赖项以使 cloudera 上游版本正常工作吗？
看起来你已经设置了这个 sbt-plugin
com.typesafe.sbt:sbt-pgp:1.0-M1
在你的全局 sbt 配置中
C:\Users\avi\.sbt.0\plugins\plugins.sbt#L3-4
问题是版本 1.0-M1 很旧，与 sbt 1.0 和 scala 2.12
您应该将插件升级到更新的版本：
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.1")