无法使用 pickle 和多个模块加载文件
Unable to load files using pickle and multiple modules
我正在尝试创建一个使用设置和 Gui 模块的用户系统，当 GUI 模块请求使用 pickle 加载文件时，我不断收到属性错误。这是来自设置模块：
import pickle
import hashlib
class User(object):
def __init__(self, fname, lname, dob, gender):
self.firstname = fname
self.lastname = lname
self._dob = dob
self.gender = gender
self.type = 'General'
self._username = ''
self._hashkey = ''
def Report(self):
print("Full Name: {0} {1}\nDate of Birth: {2}\nGender: {3}\nAccess Level: {4}".format(self.firstname,self.lastname, self._dob, self.gender, self.type))
print(self._username)
def Genusername(self):
self._username = str(str(self._dob)[:2] + self.firstname[:2] + self.lastname[:2])
saveUsers(users)
def Genhashkey(self, password):
encoded = password.encode('utf-8','strict')
return hashlib.sha256(encoded).hexdigest()
def Verifypassword(self, password):
if self._hashkey == self.Genhashkey(password):
return True
else:
return False
class SAdmin(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Stock Admin'
class Manager(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Manager'
def saveUsers(users):
with open('user_data.pkl', 'wb') as file:
pickle.dump(users, file, -1) # PICKLE HIGHEST LEVEL PROTOCOL
def loadUsers(users):
try:
with open('user_data.pkl', 'rb') as file:
temp = pickle.load(file)
for item in temp:
users.append(item)
except IOError:
saveUsers([])
def userReport(users):
for user in users:
print(user.firstname, user.lastname)
def addUser(users):
fname = input('What is your First Name?\n > ')
lname = input('What is your Last Name?\n > ')
dob = int(input('Please enter your date of birth in the following format, example 12211996\n> '))
gender = input("What is your gender? 'M' or 'F'\n >")
level = input("Enter the access level given to this user 'G', 'A', 'M'\n > ")
password = input("Enter a password:\n > ")
if level == 'G':
usertype = User
if level == 'A':
usertype = SAdmin
if level == 'M':
usertype = Manager
users.append(usertype(fname, lname, dob, gender))
user = users[len(users)-1]
user.Genusername()
user._hashkey = user.Genhashkey(password)
saveUsers(users)
def deleteUser(users):
userReport(users)
delete = input('Please type in the First Name of the user do you wish to delete:\n > ')
for user in users:
if user.firstname == delete:
users.remove(user)
saveUsers(users)
def changePass(users):
userReport(users)
change = input('Please type in the First Name of the user you wish to change the password for :\n > ')
for user in users:
if user.firstname == change:
oldpass = input('Please type in your old password:\n > ')
newpass = input('Please type in your new password:\n > ')
if user.Verifypassword(oldpass):
user._hashkey = user.Genhashkey(newpass)
saveUsers(users)
else:
print('Your old password does not match!')
def verifyUser(username, password):
for user in users:
if user._username == username and user.Verifypassword(password):
return True
else:
return False
if __name__ == '__main__':
users = []
loadUsers(users)
这是 GUI 模块：
from PyQt4 import QtGui, QtCore
import Settings
class loginWindow(QtGui.QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.lbl1 = QtGui.QLabel('Username')
self.lbl2 = QtGui.QLabel('Password')
self.username = QtGui.QLineEdit()
self.password = QtGui.QLineEdit()
self.okButton = QtGui.QPushButton("OK")
self.okButton.clicked.connect(self.tryLogin)
self.cancelButton = QtGui.QPushButton("Cancel")
grid = QtGui.QGridLayout()
grid.setSpacing(10)
grid.addWidget(self.lbl1, 1, 0)
grid.addWidget(self.username, 1, 1)
grid.addWidget(self.lbl2, 2, 0)
grid.addWidget(self.password, 2, 1)
grid.addWidget(self.okButton, 3, 1)
grid.addWidget(self.cancelButton, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 2950, 150)
self.setWindowTitle('Login')
self.show()
def tryLogin(self):
print(self.username.text(), self.password.text())
if Settings.verifyUser(self.username.text(),self.password.text()):
print('it Woks')
else:
QtGui.QMessageBox.warning(
self, 'Error', 'Incorrect Username or Password')
class Window(QtGui.QMainWindow):
def __init__(self):
super().__init__()
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
users = []
Settings.loadUsers(users)
if loginWindow().exec_() == QtGui.QDialog.Accepted:
window = Window()
window.show()
sys.exit(app.exec_())
每个用户都是一个 class 并被放入一个列表中，然后当我只加载设置文件并验证登录时使用 pickle 保存列表，但当我打开 GUI 时一切正常模块并尝试验证它不允许我，我得到的错误：
Traceback (most recent call last):
File "C:\Users`Program\LoginGUI.py", line 53, in <module>
Settings.loadUsers(users)
File "C:\Users\Program\Settings.py", line 51, in loadUsers
temp = pickle.load(file)
AttributeError: Can't get attribute 'Manager' on <module '__main__' (built-in)>
问题是您 pickling 设置中定义的对象实际上是 运行ning 'Settings' 模块，然后您试图取消 pickle来自 GUI 模块的对象。
请记住，pickle 实际上并不存储有关 class/object 是如何构造的信息，并且在解包时需要访问 class。有关详细信息，请参阅 wiki on using Pickle。
在 pkl 数据中，您看到被引用的对象是 __main__.Manager，因为在您创建 pickle 文件时 'Settings' 模块是 main （即你 运行 'Settings' 模块作为调用 addUser 函数的主脚本。
然后，您尝试在 'Gui' 中进行 unpickling - 因此该模块的名称为 __main__，并且您正在该模块中导入设置。所以经理 class 当然实际上是 Settings.Manager。但是 pkl 文件不知道这一点，并在 __main__ 中寻找 Manager class，并抛出一个 AttributeError 因为它不存在（Settings.Manager 存在，但是 __main__.Manager 没有）。
下面是用于演示的最小代码集。
class_def.py模块：
import pickle
class Foo(object):
def __init__(self, name):
self.name = name
def main():
foo = Foo('a')
with open('test_data.pkl', 'wb') as f:
pickle.dump([foo], f, -1)
if __name__=='__main__':
main()
你运行上面的内容来生成pickle数据。
main_module.py 模块：
import pickle
import class_def
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
您 运行 尝试打开 pickle 文件，这引发了与您看到的大致相同的错误。 （略有不同，但我猜那是因为我在 Python 2.7）
解决方案是：
- 您通过显式导入使 class 在顶级模块（即 GUI 或 main_module）的命名空间内可用，或者
- 您从与您将在其中打开它的相同的顶级模块创建 pickle 文件（即从 GUI 调用
Settings.addUser，或从 main_module 调用 class_def.main） .这意味着 pkl 文件会将对象保存为 Settings.Manager 或 class_def.Foo，然后可以在 GUI`main_module` 命名空间中找到它们。
选项 1 示例：
import pickle
import class_def
from class_def import Foo # Import Foo into main_module's namespace explicitly
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
选项 2 示例：
import pickle
import class_def
if __name__=='__main__':
class_def.main() # Objects are being pickled with main_module as the top-level
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
请先阅读提到的答案，了解属性错误的原因。除了他已经提供的解决方案之外，在 python3 中，您可以使用 pickle.Unpickler class 并覆盖 find_class 方法，如下所述：
import pickle
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
if name == 'Manager':
from settings import Manager
return Manager
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
如果您在模块外定义了一个 class，其对象在 pickle 数据中，
你必须导入 class
from outside_module import DefinedClass1, DefinedClass2, DefinedClass3
with open('pickle_file.pkl', 'rb') as f:
pickle_data = pickle.load(f)
如果在加载模块 () 中导入适当的 类 后仍然出现此错误，那么 pickle.Unpickler 的 find_class 函数可以被覆盖并明确指示查看当前模块的命名空间。
import pickle
from settings import Manager
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
try:
return super().find_class(__name__, name)
except AttributeError:
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
## No exception trying to get 'Manager'
注意：此方法会丢失存储在module中的相对导入路径信息。因此，请注意 pickled 类.
中的命名空间冲突
如果你使用 dill dump/load 模型就可以了
import dill
from sklearn.preprocessing import FunctionTransformer
sp_clf = FunctionTransformer(lambda X:X.astype('float').fillna(0).applymap(abs))
with open('temp.joblib','wb') as io:
dill.dump(sp_clf,io)
with open('temp.joblib','rb') as io:
dd=dill.load(io)
我正在尝试创建一个使用设置和 Gui 模块的用户系统，当 GUI 模块请求使用 pickle 加载文件时，我不断收到属性错误。这是来自设置模块：
import pickle
import hashlib
class User(object):
def __init__(self, fname, lname, dob, gender):
self.firstname = fname
self.lastname = lname
self._dob = dob
self.gender = gender
self.type = 'General'
self._username = ''
self._hashkey = ''
def Report(self):
print("Full Name: {0} {1}\nDate of Birth: {2}\nGender: {3}\nAccess Level: {4}".format(self.firstname,self.lastname, self._dob, self.gender, self.type))
print(self._username)
def Genusername(self):
self._username = str(str(self._dob)[:2] + self.firstname[:2] + self.lastname[:2])
saveUsers(users)
def Genhashkey(self, password):
encoded = password.encode('utf-8','strict')
return hashlib.sha256(encoded).hexdigest()
def Verifypassword(self, password):
if self._hashkey == self.Genhashkey(password):
return True
else:
return False
class SAdmin(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Stock Admin'
class Manager(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Manager'
def saveUsers(users):
with open('user_data.pkl', 'wb') as file:
pickle.dump(users, file, -1) # PICKLE HIGHEST LEVEL PROTOCOL
def loadUsers(users):
try:
with open('user_data.pkl', 'rb') as file:
temp = pickle.load(file)
for item in temp:
users.append(item)
except IOError:
saveUsers([])
def userReport(users):
for user in users:
print(user.firstname, user.lastname)
def addUser(users):
fname = input('What is your First Name?\n > ')
lname = input('What is your Last Name?\n > ')
dob = int(input('Please enter your date of birth in the following format, example 12211996\n> '))
gender = input("What is your gender? 'M' or 'F'\n >")
level = input("Enter the access level given to this user 'G', 'A', 'M'\n > ")
password = input("Enter a password:\n > ")
if level == 'G':
usertype = User
if level == 'A':
usertype = SAdmin
if level == 'M':
usertype = Manager
users.append(usertype(fname, lname, dob, gender))
user = users[len(users)-1]
user.Genusername()
user._hashkey = user.Genhashkey(password)
saveUsers(users)
def deleteUser(users):
userReport(users)
delete = input('Please type in the First Name of the user do you wish to delete:\n > ')
for user in users:
if user.firstname == delete:
users.remove(user)
saveUsers(users)
def changePass(users):
userReport(users)
change = input('Please type in the First Name of the user you wish to change the password for :\n > ')
for user in users:
if user.firstname == change:
oldpass = input('Please type in your old password:\n > ')
newpass = input('Please type in your new password:\n > ')
if user.Verifypassword(oldpass):
user._hashkey = user.Genhashkey(newpass)
saveUsers(users)
else:
print('Your old password does not match!')
def verifyUser(username, password):
for user in users:
if user._username == username and user.Verifypassword(password):
return True
else:
return False
if __name__ == '__main__':
users = []
loadUsers(users)
这是 GUI 模块：
from PyQt4 import QtGui, QtCore
import Settings
class loginWindow(QtGui.QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.lbl1 = QtGui.QLabel('Username')
self.lbl2 = QtGui.QLabel('Password')
self.username = QtGui.QLineEdit()
self.password = QtGui.QLineEdit()
self.okButton = QtGui.QPushButton("OK")
self.okButton.clicked.connect(self.tryLogin)
self.cancelButton = QtGui.QPushButton("Cancel")
grid = QtGui.QGridLayout()
grid.setSpacing(10)
grid.addWidget(self.lbl1, 1, 0)
grid.addWidget(self.username, 1, 1)
grid.addWidget(self.lbl2, 2, 0)
grid.addWidget(self.password, 2, 1)
grid.addWidget(self.okButton, 3, 1)
grid.addWidget(self.cancelButton, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 2950, 150)
self.setWindowTitle('Login')
self.show()
def tryLogin(self):
print(self.username.text(), self.password.text())
if Settings.verifyUser(self.username.text(),self.password.text()):
print('it Woks')
else:
QtGui.QMessageBox.warning(
self, 'Error', 'Incorrect Username or Password')
class Window(QtGui.QMainWindow):
def __init__(self):
super().__init__()
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
users = []
Settings.loadUsers(users)
if loginWindow().exec_() == QtGui.QDialog.Accepted:
window = Window()
window.show()
sys.exit(app.exec_())
每个用户都是一个 class 并被放入一个列表中，然后当我只加载设置文件并验证登录时使用 pickle 保存列表，但当我打开 GUI 时一切正常模块并尝试验证它不允许我，我得到的错误：
Traceback (most recent call last):
File "C:\Users`Program\LoginGUI.py", line 53, in <module>
Settings.loadUsers(users)
File "C:\Users\Program\Settings.py", line 51, in loadUsers
temp = pickle.load(file)
AttributeError: Can't get attribute 'Manager' on <module '__main__' (built-in)>
问题是您 pickling 设置中定义的对象实际上是 运行ning 'Settings' 模块，然后您试图取消 pickle来自 GUI 模块的对象。
请记住，pickle 实际上并不存储有关 class/object 是如何构造的信息，并且在解包时需要访问 class。有关详细信息，请参阅 wiki on using Pickle。
在 pkl 数据中，您看到被引用的对象是 __main__.Manager，因为在您创建 pickle 文件时 'Settings' 模块是 main （即你 运行 'Settings' 模块作为调用 addUser 函数的主脚本。
然后，您尝试在 'Gui' 中进行 unpickling - 因此该模块的名称为 __main__，并且您正在该模块中导入设置。所以经理 class 当然实际上是 Settings.Manager。但是 pkl 文件不知道这一点，并在 __main__ 中寻找 Manager class，并抛出一个 AttributeError 因为它不存在（Settings.Manager 存在，但是 __main__.Manager 没有）。
下面是用于演示的最小代码集。
class_def.py模块：
import pickle
class Foo(object):
def __init__(self, name):
self.name = name
def main():
foo = Foo('a')
with open('test_data.pkl', 'wb') as f:
pickle.dump([foo], f, -1)
if __name__=='__main__':
main()
你运行上面的内容来生成pickle数据。
main_module.py 模块：
import pickle
import class_def
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
您 运行 尝试打开 pickle 文件，这引发了与您看到的大致相同的错误。 （略有不同，但我猜那是因为我在 Python 2.7）
解决方案是：
- 您通过显式导入使 class 在顶级模块（即 GUI 或 main_module）的命名空间内可用，或者
- 您从与您将在其中打开它的相同的顶级模块创建 pickle 文件（即从 GUI 调用
Settings.addUser，或从 main_module 调用class_def.main） .这意味着 pkl 文件会将对象保存为Settings.Manager或class_def.Foo，然后可以在GUI`main_module` 命名空间中找到它们。
选项 1 示例：
import pickle
import class_def
from class_def import Foo # Import Foo into main_module's namespace explicitly
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
选项 2 示例：
import pickle
import class_def
if __name__=='__main__':
class_def.main() # Objects are being pickled with main_module as the top-level
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
请先阅读python3 中，您可以使用 pickle.Unpickler class 并覆盖 find_class 方法，如下所述：
import pickle
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
if name == 'Manager':
from settings import Manager
return Manager
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
如果您在模块外定义了一个 class，其对象在 pickle 数据中， 你必须导入 class
from outside_module import DefinedClass1, DefinedClass2, DefinedClass3
with open('pickle_file.pkl', 'rb') as f:
pickle_data = pickle.load(f)
如果在加载模块 (pickle.Unpickler 的 find_class 函数可以被覆盖并明确指示查看当前模块的命名空间。
import pickle
from settings import Manager
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
try:
return super().find_class(__name__, name)
except AttributeError:
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
## No exception trying to get 'Manager'
注意：此方法会丢失存储在module中的相对导入路径信息。因此，请注意 pickled 类.
如果你使用 dill dump/load 模型就可以了
import dill
from sklearn.preprocessing import FunctionTransformer
sp_clf = FunctionTransformer(lambda X:X.astype('float').fillna(0).applymap(abs))
with open('temp.joblib','wb') as io:
dill.dump(sp_clf,io)
with open('temp.joblib','rb') as io:
dd=dill.load(io)