UnpicklingError: A load persistent id instruction was encountered, but no persistent_load function was specified

Question

我正在尝试运行一个名为 api.py 的 python 文件。在此文件中，我正在加载使用 PyTorch 构建和训练的深度学习模型的 pickle 文件。

api.py 在 api.py 中，下面给出的函数是最重要的函数。

def load_model_weights(model_architecture, weights_path):
  if os.path.isfile(weights_path):
      cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
      model_architecture.load_state_dict(torch.load(weights_path))
  else:
      raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

目录结构

MP1
 ┣ .ipynb_checkpoints
 ┃ ┗ RS_netflix3months_100epochs_64,128,128-checkpoint.ipynb
 ┣ data
 ┃ ┣ AutoEncoder.png
 ┃ ┣ collaborative_filtering.gif
 ┃ ┣ movie_titles.txt
 ┃ ┗ shut_up.gif
 ┣ DeepRecommender
 ┃ ┣ data_utils
 ┃ ┃ ┣ movielens_data_convert.py
 ┃ ┃ ┗ netflix_data_convert.py
 ┃ ┣ reco_encoder
 ┃ ┃ ┣ data
 ┃ ┃ ┃ ┣ __pycache__
 ┃ ┃ ┃ ┃ ┣ input_layer.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┣ input_layer_api.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ __init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ input_layer.py
 ┃ ┃ ┃ ┣ input_layer_api.py
 ┃ ┃ ┃ ┗ __init__.py
 ┃ ┃ ┣ model
 ┃ ┃ ┃ ┣ __pycache__
 ┃ ┃ ┃ ┃ ┣ model.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ __init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ model.py
 ┃ ┃ ┃ ┗ __init__.py
 ┃ ┃ ┣ __pycache__
 ┃ ┃ ┃ ┗ __init__.cpython-37.pyc
 ┃ ┃ ┗ __init__.py
 ┃ ┣ __pycache__
 ┃ ┃ ┗ __init__.cpython-37.pyc
 ┃ ┣ compute_RMSE.py
 ┃ ┣ infer.py
 ┃ ┣ run.py
 ┃ ┗ __init__.py
 ┣ model_save
 ┃ ┣ model.epoch_99
 ┃ ┃ ┗ archive
 ┃ ┃ ┃ ┣ data
 ┃ ┃ ┃ ┃ ┣ 92901648
 ┃ ┃ ┃ ┃ ┣ 92901728
 ┃ ┃ ┃ ┃ ┣ 92901808
 ┃ ┃ ┃ ┃ ┣ 92901888
 ┃ ┃ ┃ ┃ ┣ 92901968
 ┃ ┃ ┃ ┃ ┣ 92902048
 ┃ ┃ ┃ ┃ ┣ 92902128
 ┃ ┃ ┃ ┃ ┣ 92902208
 ┃ ┃ ┃ ┃ ┣ 92902288
 ┃ ┃ ┃ ┃ ┣ 92902368
 ┃ ┃ ┃ ┃ ┣ 92902448
 ┃ ┃ ┃ ┃ ┗ 92902608
 ┃ ┃ ┃ ┣ data.pkl
 ┃ ┃ ┃ ┗ version
 ┃ ┣ model.epoch_99.zip
 ┃ ┗ model.onnx
 ┣ Netflix
 ┃ ┣ N1Y_TEST
 ┃ ┃ ┗ n1y.test.txt
 ┃ ┣ N1Y_TRAIN
 ┃ ┃ ┗ n1y.train.txt
 ┃ ┣ N1Y_VALID
 ┃ ┃ ┗ n1y.valid.txt
 ┃ ┣ N3M_TEST
 ┃ ┃ ┗ n3m.test.txt
 ┃ ┣ N3M_TRAIN
 ┃ ┃ ┗ n3m.train.txt
 ┃ ┣ N3M_VALID
 ┃ ┃ ┗ n3m.valid.txt
 ┃ ┣ N6M_TEST
 ┃ ┃ ┗ n6m.test.txt
 ┃ ┣ N6M_TRAIN
 ┃ ┃ ┗ n6m.train.txt
 ┃ ┣ N6M_VALID
 ┃ ┃ ┗ n6m.valid.txt
 ┃ ┣ NF_TEST
 ┃ ┃ ┗ nf.test.txt
 ┃ ┣ NF_TRAIN
 ┃ ┃ ┗ nf.train.txt
 ┃ ┗ NF_VALID
 ┃ ┃ ┗ nf.valid.txt
 ┣ test
 ┃ ┣ testData_iRec
 ┃ ┃ ┣ .part-00199-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt.crc
 ┃ ┃ ┣ part-00000-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┣ part-00003-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┗ _SUCCESS
 ┃ ┣ testData_uRec
 ┃ ┃ ┣ .part-00000-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt.crc
 ┃ ┃ ┣ ._SUCCESS.crc
 ┃ ┃ ┣ part-00161-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┣ part-00196-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┗ part-00199-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┣ data_layer_tests.py
 ┃ ┣ test_model.py
 ┃ ┗ __init__.py
 ┣ __pycache__
 ┃ ┣ api.cpython-37.pyc
 ┃ ┣ load_test.cpython-37.pyc
 ┃ ┣ parameters.cpython-37.pyc
 ┃ ┗ utils.cpython-37.pyc
 ┣ api.py
 ┣ compute_RMSE.py
 ┣ load_test.py
 ┣ logger.py
 ┣ netflix_1y_test.csv
 ┣ netflix_1y_train.csv
 ┣ netflix_1y_valid.csv
 ┣ netflix_3m_test.csv
 ┣ netflix_3m_train.csv
 ┣ netflix_3m_valid.csv
 ┣ netflix_6m_test.csv
 ┣ netflix_6m_train.csv
 ┣ netflix_6m_valid.csv
 ┣ netflix_full_test.csv
 ┣ netflix_full_train.csv
 ┣ netflix_full_valid.csv
 ┣ parameters.py
 ┣ preds.txt
 ┣ RS_netflix3months_100epochs_64,128,128.ipynb
 ┗ utils.py

我遇到这样的错误 (serialization.py)。有人可以帮我解决这个错误吗？

D:\Anaconda\envs\practise\lib\site-packages\torch\serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763 
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

Answer 1

在搜索 PyTorch 文档后，我最终将模型保存为 ONNX 格式，然后将该 ONNX 模型加载到 PyTorch 模型中并将其用于推理。

import onnx
from onnx2pytorch import ConvertModel


def load_model_weights(model_architecture, weights_path):
    if os.path.isfile("model.onnx"):
        cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
        onnx_model = onnx.load("model.onnx")
        pytorch_model = ConvertModel(onnx_model)
        ## model_architecture.load_state_dict(torch.load(weights_path))
    else:
        raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

一些有用的资源：

torch.save

torch.load

ONNX tutorials

UnpicklingError: A load persistent id instruction was encountered, but no persistent_load function was specified

UnpicklingError: A load persistent id instruction was encountered, but no persistent_load function was specified

python

serialization

pickle

deep-learning

pytorch