在设备上训练声音分类器
Training sound classifier on device
我正尝试在 iOS 上的设备上训练 CoreML 声音分类器,我一直在努力寻找有关该主题的学习资源。声音分类器用于确定一段音乐是否与其他歌曲的集合相似。因此分类器的输出只是“匹配”/“不匹配”的标签。
使用 CreateML 应用程序工作流程进行训练非常简单。我只是想在 iOS 中尝试在设备上进行相同类型的培训,但据我所知(如果我错了请纠正我)iOS 不支持 createML。
我一直在尝试改编来自各种来源的代码,以使其在 iOS 操场上运行。我只能找到关于训练图像分类器的资源,这两个是最有帮助的 (1, 2)。
请在下面查看我到目前为止提出的代码。
import UIKit
import CoreML
func convertDataToArray<T>(count: Int, data: Data) -> [T] {
let array = data.withUnsafeBytes { (pointer: UnsafePointer<T>) -> [T] in
let buffer = UnsafeBufferPointer(start: pointer, count: count / MemoryLayout<Float32>.size)
return Array<T>(buffer)
}
return array
}
// Get files (names and paths) in directory
public func getAllFilesInDirectory(bundle: Bundle, directory: String, extensionWanted: String) -> (names: [String], paths: [URL]) {
let cachesURL = URL(fileURLWithPath: "/Users/...../Playgrounds/MLPlayground.playground/Resources")
let directoryURL = cachesURL.appendingPathComponent(directory)
do {
try FileManager.default.createDirectory(atPath: directoryURL.relativePath, withIntermediateDirectories: true)
// Get the directory contents urls (including subfolders urls)
let directoryContents = try FileManager.default.contentsOfDirectory(at: directoryURL, includingPropertiesForKeys: nil, options: [])
// Filter the directory contents
let filesPath = directoryContents.filter{ [=10=].pathExtension == extensionWanted }
let fileNames = filesPath.map{ [=10=].deletingPathExtension().lastPathComponent }
return (names: fileNames, paths: filesPath);
} catch {
print("Failed to fetch contents of directory: \(error.localizedDescription)")
}
return (names: [], paths: [])
}
let bundle = Bundle.main
var featureProviders = [MLFeatureProvider]()
let matchDir = getAllFilesInDirectory(bundle: bundle, directory: "Match", extensionWanted: "m4a")
let noMatchDir = getAllFilesInDirectory(bundle: bundle, directory: "No Match", extensionWanted: "m4a")
// I have ommited the full path directories for Stack Overflow
try! MLModel.compileModel(at: URL(fileURLWithPath: "/Users/...../Playgrounds/MLPlayground.playground/Resources/UpdateableML.mlmodel"))
let modelDir = URL(fileURLWithPath: "/Users/....../Playgrounds/MLPlayground.playground/Resources/UpdateableML.mlmodel")
let outputDir = URL(fileURLWithPath: "/Users/....../Playgrounds/MLPlayground.playground/Resources/Output/outputmodel.mlmodel")
func getFeatureProvider(forLabel: String, directory: URL) {
let data = try! Data(contentsOf: directory.appendingPathComponent("\(forLabel).m4a"))
// MultiArray (Float32 15600)
let mlInputData = try! MLMultiArray(shape: [15600], dataType: .float32)
let songDataArray: [Float32] = convertDataToArray(count: data.count, data: data)
let count = songDataArray.count
for i in 0..<mlInputData.count {
mlInputData[i] = NSNumber(value: songDataArray[i])
}
let soundValue = MLFeatureValue(multiArray: mlInputData)
let outputValue = MLFeatureValue(string: forLabel)
let dataPointFeatures: [String: MLFeatureValue] = ["audioSamples": soundValue, "classLabel": outputValue]
if let provider = try? MLDictionaryFeatureProvider(dictionary: dataPointFeatures) {
featureProviders.append(provider)
} else {
print("Failed to get provider")
}
}
// Get features
for s in matchDir.names {
getFeatureProvider(forLabel: s, directory: matchDir.paths.first!.deletingLastPathComponent())
}
for s in noMatchDir.names {
getFeatureProvider(forLabel: s, directory: noMatchDir.paths.first!.deletingLastPathComponent())
}
var batchProvider = MLArrayBatchProvider(array: featureProviders)
func updateModel(at url: URL, with trainingData: MLBatchProvider, completionHandler: @escaping (MLUpdateContext) -> Void) {
let updateTask = try! MLUpdateTask(
forModelAt: url,
trainingData: trainingData,
configuration: nil,
completionHandler: completionHandler
)
updateTask.resume()
}
func saveUpdatedModel(_ updateContext: MLUpdateContext) {
let updatedModel = updateContext.model
let fileManager = FileManager.default
do {
try fileManager.createDirectory(
at: outputDir,
withIntermediateDirectories: true,
attributes: nil)
try updatedModel.write(to: outputDir)
print("Updated model saved to:\n\t\(outputDir)")
} catch let error {
print("Could not save updated model to the file system: \(error)")
return
}
}
func updateWith(trainingData: MLBatchProvider, completionHandler: @escaping () -> Void) {
updateModel(at: modelDir, with: trainingData) { context in
print("Update Complete")
saveUpdatedModel(context)
completionHandler()
}
}
updateWith(trainingData: batchProvider, completionHandler: {
print("Final Complete")
})
我现在有两个问题:
- 我从函数 'updateModel' 的 MLUpdateTask 收到以下错误:
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=0 "Unable to load model at file:///Users/....../Playgrounds/CuratorMLPlayground.playground/Resources/UpdateableML.mlmodel with error: Error opening file stream: /Users/....../Playgrounds/CuratorMLPlayground.playground/Resources/UpdateableML.mlmodel/coremldata.bin: unspecified iostream_category error"
- 我不知道我是否在 'getFeatureProvider' 函数中正确获取了音频数据,因为 'songDataArray' 的大小大约是 260000 和模型的形状/'mlInputData'是 15600 吗?谁能给我解释一下。
更新:
我已将其复制到我实际的 iOS 应用程序项目中。我现在收到以下错误,而不是上面的错误。
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=0 "Invalid URL for .mlmodel." UserInfo={NSLocalizedDescription=Invalid URL for .mlmodel.}:
但是,我几乎可以肯定 URL 正确指向 mlmodel
我已经设法解决了与 mlUpdate 任务相关的错误,问题是我引用的是 .mlmodel 而不是编译版本,即 .mlmodelc 。从 Xcode 构建 iOS 应用程序时,会自动生成此文件。
我现在收到以下错误:
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=6 "Pipeline is not marked as updatable to perform update." UserInfo={NSLocalizedDescription=Pipeline is not marked as updatable to perform update.}:
因此,我可以得出结论,现在只是建立一个更好的模型的问题。我现在假设如果我有合适的模型,updating/personalising 设备上的代码就可以工作。
因此,现在只需构建一个适用于此的模型即可。感谢another answer by Matthjis,我现在意识到我在 CreateML 中制作的模型无法更新,因为它是一个 GLM 分类器。
我想我也发现了在 swift 中加载音频数据的正确方法,感谢 this git repo。
我正尝试在 iOS 上的设备上训练 CoreML 声音分类器,我一直在努力寻找有关该主题的学习资源。声音分类器用于确定一段音乐是否与其他歌曲的集合相似。因此分类器的输出只是“匹配”/“不匹配”的标签。
使用 CreateML 应用程序工作流程进行训练非常简单。我只是想在 iOS 中尝试在设备上进行相同类型的培训,但据我所知(如果我错了请纠正我)iOS 不支持 createML。
我一直在尝试改编来自各种来源的代码,以使其在 iOS 操场上运行。我只能找到关于训练图像分类器的资源,这两个是最有帮助的 (1, 2)。
请在下面查看我到目前为止提出的代码。
import UIKit
import CoreML
func convertDataToArray<T>(count: Int, data: Data) -> [T] {
let array = data.withUnsafeBytes { (pointer: UnsafePointer<T>) -> [T] in
let buffer = UnsafeBufferPointer(start: pointer, count: count / MemoryLayout<Float32>.size)
return Array<T>(buffer)
}
return array
}
// Get files (names and paths) in directory
public func getAllFilesInDirectory(bundle: Bundle, directory: String, extensionWanted: String) -> (names: [String], paths: [URL]) {
let cachesURL = URL(fileURLWithPath: "/Users/...../Playgrounds/MLPlayground.playground/Resources")
let directoryURL = cachesURL.appendingPathComponent(directory)
do {
try FileManager.default.createDirectory(atPath: directoryURL.relativePath, withIntermediateDirectories: true)
// Get the directory contents urls (including subfolders urls)
let directoryContents = try FileManager.default.contentsOfDirectory(at: directoryURL, includingPropertiesForKeys: nil, options: [])
// Filter the directory contents
let filesPath = directoryContents.filter{ [=10=].pathExtension == extensionWanted }
let fileNames = filesPath.map{ [=10=].deletingPathExtension().lastPathComponent }
return (names: fileNames, paths: filesPath);
} catch {
print("Failed to fetch contents of directory: \(error.localizedDescription)")
}
return (names: [], paths: [])
}
let bundle = Bundle.main
var featureProviders = [MLFeatureProvider]()
let matchDir = getAllFilesInDirectory(bundle: bundle, directory: "Match", extensionWanted: "m4a")
let noMatchDir = getAllFilesInDirectory(bundle: bundle, directory: "No Match", extensionWanted: "m4a")
// I have ommited the full path directories for Stack Overflow
try! MLModel.compileModel(at: URL(fileURLWithPath: "/Users/...../Playgrounds/MLPlayground.playground/Resources/UpdateableML.mlmodel"))
let modelDir = URL(fileURLWithPath: "/Users/....../Playgrounds/MLPlayground.playground/Resources/UpdateableML.mlmodel")
let outputDir = URL(fileURLWithPath: "/Users/....../Playgrounds/MLPlayground.playground/Resources/Output/outputmodel.mlmodel")
func getFeatureProvider(forLabel: String, directory: URL) {
let data = try! Data(contentsOf: directory.appendingPathComponent("\(forLabel).m4a"))
// MultiArray (Float32 15600)
let mlInputData = try! MLMultiArray(shape: [15600], dataType: .float32)
let songDataArray: [Float32] = convertDataToArray(count: data.count, data: data)
let count = songDataArray.count
for i in 0..<mlInputData.count {
mlInputData[i] = NSNumber(value: songDataArray[i])
}
let soundValue = MLFeatureValue(multiArray: mlInputData)
let outputValue = MLFeatureValue(string: forLabel)
let dataPointFeatures: [String: MLFeatureValue] = ["audioSamples": soundValue, "classLabel": outputValue]
if let provider = try? MLDictionaryFeatureProvider(dictionary: dataPointFeatures) {
featureProviders.append(provider)
} else {
print("Failed to get provider")
}
}
// Get features
for s in matchDir.names {
getFeatureProvider(forLabel: s, directory: matchDir.paths.first!.deletingLastPathComponent())
}
for s in noMatchDir.names {
getFeatureProvider(forLabel: s, directory: noMatchDir.paths.first!.deletingLastPathComponent())
}
var batchProvider = MLArrayBatchProvider(array: featureProviders)
func updateModel(at url: URL, with trainingData: MLBatchProvider, completionHandler: @escaping (MLUpdateContext) -> Void) {
let updateTask = try! MLUpdateTask(
forModelAt: url,
trainingData: trainingData,
configuration: nil,
completionHandler: completionHandler
)
updateTask.resume()
}
func saveUpdatedModel(_ updateContext: MLUpdateContext) {
let updatedModel = updateContext.model
let fileManager = FileManager.default
do {
try fileManager.createDirectory(
at: outputDir,
withIntermediateDirectories: true,
attributes: nil)
try updatedModel.write(to: outputDir)
print("Updated model saved to:\n\t\(outputDir)")
} catch let error {
print("Could not save updated model to the file system: \(error)")
return
}
}
func updateWith(trainingData: MLBatchProvider, completionHandler: @escaping () -> Void) {
updateModel(at: modelDir, with: trainingData) { context in
print("Update Complete")
saveUpdatedModel(context)
completionHandler()
}
}
updateWith(trainingData: batchProvider, completionHandler: {
print("Final Complete")
})
我现在有两个问题:
- 我从函数 'updateModel' 的 MLUpdateTask 收到以下错误:
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=0 "Unable to load model at file:///Users/....../Playgrounds/CuratorMLPlayground.playground/Resources/UpdateableML.mlmodel with error: Error opening file stream: /Users/....../Playgrounds/CuratorMLPlayground.playground/Resources/UpdateableML.mlmodel/coremldata.bin: unspecified iostream_category error"
- 我不知道我是否在 'getFeatureProvider' 函数中正确获取了音频数据,因为 'songDataArray' 的大小大约是 260000 和模型的形状/'mlInputData'是 15600 吗?谁能给我解释一下。
更新: 我已将其复制到我实际的 iOS 应用程序项目中。我现在收到以下错误,而不是上面的错误。
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=0 "Invalid URL for .mlmodel." UserInfo={NSLocalizedDescription=Invalid URL for .mlmodel.}:
但是,我几乎可以肯定 URL 正确指向 mlmodel
我已经设法解决了与 mlUpdate 任务相关的错误,问题是我引用的是 .mlmodel 而不是编译版本,即 .mlmodelc 。从 Xcode 构建 iOS 应用程序时,会自动生成此文件。
我现在收到以下错误:
Fatal error: 'try!' expression unexpectedly raised an error: Error Domain=com.apple.CoreML Code=6 "Pipeline is not marked as updatable to perform update." UserInfo={NSLocalizedDescription=Pipeline is not marked as updatable to perform update.}:
因此,我可以得出结论,现在只是建立一个更好的模型的问题。我现在假设如果我有合适的模型,updating/personalising 设备上的代码就可以工作。
因此,现在只需构建一个适用于此的模型即可。感谢another answer by Matthjis,我现在意识到我在 CreateML 中制作的模型无法更新,因为它是一个 GLM 分类器。
我想我也发现了在 swift 中加载音频数据的正确方法,感谢 this git repo。