Spark ML - KMeans - org.apache.spark.sql.AnalysisException：无法解析给定输入列的“特征”
Spark ML - KMeans - org.apache.spark.sql.AnalysisException: cannot resolve '`features`' given input columns
我正在尝试使用 Spark ML KMeans 分析和聚类芝加哥犯罪数据集。下面是片段
case class ChicCase(ID: Long, Case_Number: String, Date: String, Block: String, IUCR: String, Primary_Type: String, Description: String, Location_description: String, Arrest: Boolean, Domestic: Boolean, Beat: Int, District: Int, Ward: Int, Community_Area: Int, FBI_Code: String, X_Coordinate: Int, Y_Coordinate: Int, Year: Int, Updated_On: String, Latitude: Double, Longitude: Double, Location: String)
val city = spark.read.option("header", true).option("inferSchema", true).csv("/chicago_city/Crimes_2001_to_present_2").as[ChicCase]
val data = city.drop("ID", "Case_Number", "Date", "Block", "IUCR", "Primary_Type", "Description", "Location_description", "Arrest", "Domestic", "FBI_Code", "Year", "Location", "Updated_On")
val kmeans = new KMeans
kmeans.setK(10).setSeed(1L)
val model = kmeans.fit(data)
但这是抛出以下异常
org.apache.spark.sql.AnalysisException: cannot resolve '`features`' given input columns: [Ward, Longitude, X_Coordinate, Beat, Latitude, District, Y_Coordinate, Community_Area];
at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$$anonfun$apply.applyOrElse(CheckAnalysis.scala:77)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$$anonfun$apply.applyOrElse(CheckAnalysis.scala:74)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp.apply(TreeNode.scala:301)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp.apply(TreeNode.scala:301)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:300)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp(QueryPlan.scala:190)
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform(QueryPlan.scala:200)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform.apply(QueryPlan.scala:204)
at scala.collection.TraversableLike$$anonfun$map.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map.apply(TraversableLike.scala:234)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform(QueryPlan.scala:204)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun.apply(QueryPlan.scala:209)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:209)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis.apply(CheckAnalysis.scala:74)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis.apply(CheckAnalysis.scala:67)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67)
at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:58)
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2589)
at org.apache.spark.sql.Dataset.select(Dataset.scala:969)
at org.apache.spark.ml.clustering.KMeans.fit(KMeans.scala:307) ... 90 elided
数据类型为 Int 或 Double。可能是什么问题？
在 spark ml 数据框 API 中，应使用带有特征名称的 VectorAssembler 将所有特征列收集为一列。当你拟合模型时，它会尝试找到 features 列，在你的情况下，没有这样的列，这就是为什么 Exception: cannot resolve 'features' 给定输入列:
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.clustering.KMeans
// assembler to collect all interesting columns into a single features column
val assembler = (new VectorAssembler().
setInputCols(Array("Ward", "Longitude", "X_Coordinate", "Beat",
"Latitude", "District", "Y_Coordinate",
"Community_Area")).
setOutputCol("features"))
val data = assembler.transform(city)
val kmeans = new KMeans()
val model = kmeans.fit(data)
model.getK
// res28: Int = 2 example here
我正在尝试使用 Spark ML KMeans 分析和聚类芝加哥犯罪数据集。下面是片段
case class ChicCase(ID: Long, Case_Number: String, Date: String, Block: String, IUCR: String, Primary_Type: String, Description: String, Location_description: String, Arrest: Boolean, Domestic: Boolean, Beat: Int, District: Int, Ward: Int, Community_Area: Int, FBI_Code: String, X_Coordinate: Int, Y_Coordinate: Int, Year: Int, Updated_On: String, Latitude: Double, Longitude: Double, Location: String)
val city = spark.read.option("header", true).option("inferSchema", true).csv("/chicago_city/Crimes_2001_to_present_2").as[ChicCase]
val data = city.drop("ID", "Case_Number", "Date", "Block", "IUCR", "Primary_Type", "Description", "Location_description", "Arrest", "Domestic", "FBI_Code", "Year", "Location", "Updated_On")
val kmeans = new KMeans
kmeans.setK(10).setSeed(1L)
val model = kmeans.fit(data)
但这是抛出以下异常
org.apache.spark.sql.AnalysisException: cannot resolve '`features`' given input columns: [Ward, Longitude, X_Coordinate, Beat, Latitude, District, Y_Coordinate, Community_Area];
at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$$anonfun$apply.applyOrElse(CheckAnalysis.scala:77)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$$anonfun$apply.applyOrElse(CheckAnalysis.scala:74)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp.apply(TreeNode.scala:301)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp.apply(TreeNode.scala:301)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:300)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionUp(QueryPlan.scala:190)
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform(QueryPlan.scala:200)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform.apply(QueryPlan.scala:204)
at scala.collection.TraversableLike$$anonfun$map.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map.apply(TraversableLike.scala:234)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform(QueryPlan.scala:204)
at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun.apply(QueryPlan.scala:209)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179)
at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:209)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis.apply(CheckAnalysis.scala:74)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis.apply(CheckAnalysis.scala:67)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:67)
at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:58)
at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:49)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:64)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withPlan(Dataset.scala:2589)
at org.apache.spark.sql.Dataset.select(Dataset.scala:969)
at org.apache.spark.ml.clustering.KMeans.fit(KMeans.scala:307) ... 90 elided
数据类型为 Int 或 Double。可能是什么问题？
在 spark ml 数据框 API 中，应使用带有特征名称的 VectorAssembler 将所有特征列收集为一列。当你拟合模型时，它会尝试找到 features 列，在你的情况下，没有这样的列，这就是为什么 Exception: cannot resolve 'features' 给定输入列:
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.clustering.KMeans
// assembler to collect all interesting columns into a single features column
val assembler = (new VectorAssembler().
setInputCols(Array("Ward", "Longitude", "X_Coordinate", "Beat",
"Latitude", "District", "Y_Coordinate",
"Community_Area")).
setOutputCol("features"))
val data = assembler.transform(city)
val kmeans = new KMeans()
val model = kmeans.fit(data)
model.getK
// res28: Int = 2 example here