Tensorflow.Keras：自定义约束不起作用
Tensorflow.Keras: Custom Constraint Not Working
我正在尝试实施第 2.0 节中 here 显示的权重正交约束。当我尝试在 Keras 密集层上使用它时，出现值错误。
在同一篇文章的第 3.0 部分中尝试实现自定义不相关特征约束时，也会发生这种情况。
import tensorflow as tf
import numpy as np
class WeightsOrthogonalityConstraint(tf.keras.constraints.Constraint):
def __init__(self, encoding_dim, weightage = 1.0, axis = 0):
self.encoding_dim = encoding_dim
self.weightage = weightage
self.axis = axis
def weights_orthogonality(self, w):
if(self.axis==1):
w = tf.keras.backend.transpose(w)
if(self.encoding_dim > 1):
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
return self.weightage * tf.keras.backend.sqrt(tf.keras.backend.sum(tf.keras.backend.square(m)))
else:
m = tf.keras.backend.sum(w ** 2) - 1.
return m
def __call__(self, w):
return self.weights_orthogonality(w)
rand_samples = np.random.rand(16, 4)
dummy_ds = tf.data.Dataset.from_tensor_slices((rand_samples, rand_samples)).shuffle(16).batch(16)
encoder = tf.keras.layers.Dense(2, "relu", input_shape=(4,), kernel_regularizer=WeightsOrthogonalityConstraint(2))
decoder = tf.keras.layers.Dense(4, "relu")
autoencoder = tf.keras.models.Sequential()
autoencoder.add(encoder)
autoencoder.add(decoder)
autoencoder.compile(metrics=['accuracy'],
loss='mean_squared_error',
optimizer='sgd')
autoencoder.summary()
autoencoder.fit(dummy_ds, epochs=1)
如果我停止使用约束，没有错误，但是当使用时，会引发下一个错误：
2019-09-07 14:20:25.962610: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library nvcuda.dll
2019-09-07 14:20:26.997957: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties:
name: GeForce GTX 1060 major: 6 minor: 1 memoryClockRate(GHz): 1.733
pciBusID: 0000:01:00.0
2019-09-07 14:20:27.043016: I tensorflow/stream_executor/platform/default/dlopen_checker_stub.cc:25] GPU libraries are statically linked, skip dlopen check.
2019-09-07 14:20:27.050749: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0
2019-09-07 14:20:27.081369: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
2019-09-07 14:20:27.113598: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties:
name: GeForce GTX 1060 major: 6 minor: 1 memoryClockRate(GHz): 1.733
pciBusID: 0000:01:00.0
2019-09-07 14:20:27.144194: I tensorflow/stream_executor/platform/default/dlopen_checker_stub.cc:25] GPU libraries are statically linked, skip dlopen check.
2019-09-07 14:20:27.151802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0
2019-09-07 14:20:27.800616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1181] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 14:20:27.817323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1187] 0
2019-09-07 14:20:27.840635: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1200] 0: N
2019-09-07 14:20:27.848536: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1326] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 4712 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060, pci bus id: 0000:01:00.0, compute capability: 6.1)
Traceback (most recent call last):
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\ptvsd_launcher.py", line 43, in <module>
main(ptvsdArgs)
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 432, in main
run()
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 316, in run_file
runpy.run_path(target, run_name='__main__')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 35, in <module>
optimizer='sgd')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\training\tracking\base.py", line 458, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 337, in compile
self._compile_weights_loss_and_weighted_metrics()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\training\tracking\base.py", line 458, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1494, in _compile_weights_loss_and_weighted_metrics
self.total_loss = self._prepare_total_loss(masks)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1601, in _prepare_total_loss
custom_losses = self.get_losses_for(None) + self.get_losses_for(
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 1209, in get_losses_for
return [l for l in self.losses if l._unconditional_loss]
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 835, in losses
return collected_losses + self._gather_children_attribute('losses')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 2129, in _gather_children_attribute
getattr(layer, attribute) for layer in nested_layers))
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 2129, in <genexpr>
getattr(layer, attribute) for layer in nested_layers))
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 832, in losses
loss_tensor = regularizer()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 907, in _tag_unconditional
loss = loss()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 1659, in _loss_for_variable
regularization = regularizer(v)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 21, in __call__
return self.weights_orthogonality(w)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 14, in weights_orthogonality
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\backend.py", line 1310, in eye
return variable(linalg_ops.eye(size, dtype=tf_dtype), dtype, name)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\backend.py", line 785, in variable
constraint=constraint)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\variables.py", line 264, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py", line 464, in __init__
shape=shape)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py", line 550, in _init_from_args
raise ValueError("Tensor-typed variable initializers must either be "
ValueError: Tensor-typed variable initializers must either be wrapped in an init_scope or callable (e.g., `tf.Variable(lambda : tf.truncated_normal([10, 40]))`) when building functions. Please file a feature request if this restriction inconveniences you.
提前致谢！
PD：Here 是一个显示错误的 Colab Notebook
PD2：我设法找到导致问题的线路，是这个：
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
特别是 keras 后端 eye() 函数导致了问题
我用 TensorFlow 版本 1.14.0 检查了同样的情况，它工作正常。
我在下面分享我的 colab notebook link。请检查！
我设法解决了这个问题：
导致错误的函数是第 14 行的 tf.keras.backed.eye()。我在那里读到该函数在 keras 后端的实现使用 numpy 数组作为单位矩阵，但 tensorflow 和其他后端已经使用张量实现此功能。 tf2.0 上是因为缺少张量导致的错误，所以只需将 tf.keras.backed.eye() 更改为 tf.eye() 即可解决问题。
我正在尝试实施第 2.0 节中 here 显示的权重正交约束。当我尝试在 Keras 密集层上使用它时，出现值错误。
在同一篇文章的第 3.0 部分中尝试实现自定义不相关特征约束时，也会发生这种情况。
import tensorflow as tf
import numpy as np
class WeightsOrthogonalityConstraint(tf.keras.constraints.Constraint):
def __init__(self, encoding_dim, weightage = 1.0, axis = 0):
self.encoding_dim = encoding_dim
self.weightage = weightage
self.axis = axis
def weights_orthogonality(self, w):
if(self.axis==1):
w = tf.keras.backend.transpose(w)
if(self.encoding_dim > 1):
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
return self.weightage * tf.keras.backend.sqrt(tf.keras.backend.sum(tf.keras.backend.square(m)))
else:
m = tf.keras.backend.sum(w ** 2) - 1.
return m
def __call__(self, w):
return self.weights_orthogonality(w)
rand_samples = np.random.rand(16, 4)
dummy_ds = tf.data.Dataset.from_tensor_slices((rand_samples, rand_samples)).shuffle(16).batch(16)
encoder = tf.keras.layers.Dense(2, "relu", input_shape=(4,), kernel_regularizer=WeightsOrthogonalityConstraint(2))
decoder = tf.keras.layers.Dense(4, "relu")
autoencoder = tf.keras.models.Sequential()
autoencoder.add(encoder)
autoencoder.add(decoder)
autoencoder.compile(metrics=['accuracy'],
loss='mean_squared_error',
optimizer='sgd')
autoencoder.summary()
autoencoder.fit(dummy_ds, epochs=1)
如果我停止使用约束，没有错误，但是当使用时，会引发下一个错误：
2019-09-07 14:20:25.962610: I tensorflow/stream_executor/platform/default/dso_loader.cc:42] Successfully opened dynamic library nvcuda.dll
2019-09-07 14:20:26.997957: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties:
name: GeForce GTX 1060 major: 6 minor: 1 memoryClockRate(GHz): 1.733
pciBusID: 0000:01:00.0
2019-09-07 14:20:27.043016: I tensorflow/stream_executor/platform/default/dlopen_checker_stub.cc:25] GPU libraries are statically linked, skip dlopen check.
2019-09-07 14:20:27.050749: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0
2019-09-07 14:20:27.081369: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
2019-09-07 14:20:27.113598: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1640] Found device 0 with properties:
name: GeForce GTX 1060 major: 6 minor: 1 memoryClockRate(GHz): 1.733
pciBusID: 0000:01:00.0
2019-09-07 14:20:27.144194: I tensorflow/stream_executor/platform/default/dlopen_checker_stub.cc:25] GPU libraries are statically linked, skip dlopen check.
2019-09-07 14:20:27.151802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1763] Adding visible gpu devices: 0
2019-09-07 14:20:27.800616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1181] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 14:20:27.817323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1187] 0
2019-09-07 14:20:27.840635: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1200] 0: N
2019-09-07 14:20:27.848536: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1326] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 4712 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060, pci bus id: 0000:01:00.0, compute capability: 6.1)
Traceback (most recent call last):
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\ptvsd_launcher.py", line 43, in <module>
main(ptvsdArgs)
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 432, in main
run()
File "c:\Users\whitm\.vscode\extensions\ms-python.python-2019.9.34911\pythonFiles\lib\python\ptvsd\__main__.py", line 316, in run_file
runpy.run_path(target, run_name='__main__')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 35, in <module>
optimizer='sgd')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\training\tracking\base.py", line 458, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 337, in compile
self._compile_weights_loss_and_weighted_metrics()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\training\tracking\base.py", line 458, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1494, in _compile_weights_loss_and_weighted_metrics
self.total_loss = self._prepare_total_loss(masks)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1601, in _prepare_total_loss
custom_losses = self.get_losses_for(None) + self.get_losses_for(
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 1209, in get_losses_for
return [l for l in self.losses if l._unconditional_loss]
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 835, in losses
return collected_losses + self._gather_children_attribute('losses')
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 2129, in _gather_children_attribute
getattr(layer, attribute) for layer in nested_layers))
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 2129, in <genexpr>
getattr(layer, attribute) for layer in nested_layers))
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 832, in losses
loss_tensor = regularizer()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 907, in _tag_unconditional
loss = loss()
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 1659, in _loss_for_variable
regularization = regularizer(v)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 21, in __call__
return self.weights_orthogonality(w)
File "c:\Users\whitm\Desktop\CodeProjects\ForestClassifier-DEC\Test.py", line 14, in weights_orthogonality
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\backend.py", line 1310, in eye
return variable(linalg_ops.eye(size, dtype=tf_dtype), dtype, name)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\keras\backend.py", line 785, in variable
constraint=constraint)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\variables.py", line 264, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py", line 464, in __init__
shape=shape)
File "C:\ProgramData\Anaconda3\envs\tensorflow-gpu\lib\site-packages\tensorflow\python\ops\resource_variable_ops.py", line 550, in _init_from_args
raise ValueError("Tensor-typed variable initializers must either be "
ValueError: Tensor-typed variable initializers must either be wrapped in an init_scope or callable (e.g., `tf.Variable(lambda : tf.truncated_normal([10, 40]))`) when building functions. Please file a feature request if this restriction inconveniences you.
提前致谢！
PD：Here 是一个显示错误的 Colab Notebook
PD2：我设法找到导致问题的线路，是这个：
m = tf.keras.backend.dot(tf.keras.backend.transpose(w), w) - tf.keras.backend.eye(self.encoding_dim)
特别是 keras 后端 eye() 函数导致了问题
我用 TensorFlow 版本 1.14.0 检查了同样的情况，它工作正常。
我在下面分享我的 colab notebook link。请检查！
我设法解决了这个问题：
导致错误的函数是第 14 行的 tf.keras.backed.eye()。我在那里读到该函数在 keras 后端的实现使用 numpy 数组作为单位矩阵，但 tensorflow 和其他后端已经使用张量实现此功能。 tf2.0 上是因为缺少张量导致的错误，所以只需将 tf.keras.backed.eye() 更改为 tf.eye() 即可解决问题。