Python3 重新加载使用 python c-api 的项目
Python3 reload project that use python c-api
我有一个项目，我用 C (python c-api) 为它构建了一个 Class，我在 python 项目中扩展了更多。该项目的目的是为 C 库提供一个测试框架。主要针对C库的每个pull request执行测试
项目需要从Nexus服务器下载C库的相关build，编译依赖C库的python class，然后进行测试。
问题：编译C代码后导入/重新加载项目模块。
问题: 在我看来，在依赖于C库的每个函数中导入并不是那么优雅，所以我尝试调用reload，但似乎它不起作用，或者至少不像我期望的那样。
代码代码超级简化来说明问题，您可以查看此线程历史以查看以前的代码。
main.py
from utils.my_custom_py import MyCustomExtended
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
for module_name in list(sys.modules.keys()):
m = sys.modules.get(module_name)
if not hasattr(m, '__file__'):
continue
if getattr(m, '__name__', None) in [None, '__mp_main__', '__main__']:
continue
try:
# superreload(m) # from ==> IPython.extensions
# sys.modules[module_name] = reload(m)
reload(m)
except Exception as e:
...
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
utils.my_custom_py.py
from importlib.util import find_spec
if find_spec('custom2'):
import custom2
else:
class custom2:
class Custom:
...
class MyCustomExtended(custom2.Custom):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
setup.py
from distutils.core import Extension, setup
custom_ext = Extension("custom2", ["src/custom.c"])
setup(name="custom2", version="1.0", ext_modules=[custom_ext])
src.custom.c 取自：docs.python.org
错误输出:
running clean
running build
running build_ext
building 'custom2' extension
creating build
creating build/temp.linux-x86_64-3.6
creating build/temp.linux-x86_64-3.6/src
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/tmp/PlayAround/.venv/include -I/usr/include/python3.6m -c src/custom.c -o build/temp.linux-x86_64-3.6/src/custom.o
creating build/lib.linux-x86_64-3.6
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/src/custom.o -o build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so
running install
running install_lib
copying build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so -> /tmp/PlayAround/.venv/lib/python3.6/site-packages
running install_egg_info
Removing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Writing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Traceback (most recent call last):
File "./main.py", line 38, in <module>
main()
File "./main.py", line 33, in main
MyCustomExtended(1, 2, 3)
File "/tmp/PlayAround/utils/my_custom_py.py", line 12, in __init__
super().__init__(*args, **kwargs)
TypeError: object.__init__() takes no parameters
主要工作:
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
from utils.my_custom_py import MyCustomExtended
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
除了代码中的各种小问题外，您还试图重新加载所有存在的模块，同时默默地吞下错误。重新加载 utils 模块按预期工作：
import utils
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('python3 setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
for module_name in list(sys.modules.keys()):
m = sys.modules.get(module_name)
if not hasattr(m, '__file__'):
continue
if getattr(m, '__name__', None) in [None, '__mp_main__', '__main__']:
continue
reload(utils)
MyCustomExtended = utils.MyCustomExtended
print(MyCustomExtended)
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
我建议您只重新加载应用程序中的模块。
最后，我通过反复试验找到了解决方案。
对于此示例代码 id 执行了以下操作：
from utils.my_custom_py import MyCustomExtended
from importlib.util import find_spec
from importlib import reload
from sys import modules
from os import system
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def reload_my_libs():
global MyCustomExtended
reload(modules['utils'])
reload(modules['utils.my_custom_py'])
from utils.my_custom_py import MyCustomExtended
def main():
if find_spec('custom2') is None:
setup()
reload_my_libs()
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
结果，我得到了：
running clean
running build
running build_ext
building 'custom2' extension
creating build
creating build/temp.linux-x86_64-3.6
creating build/temp.linux-x86_64-3.6/src
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/tmp/PlayAround/.venv/include -I/usr/include/python3.6m -c src/custom.c -o build/temp.linux-x86_64-3.6/src/custom.o
creating build/lib.linux-x86_64-3.6
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/src/custom.o -o build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so
running install
running install_lib
copying build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so -> /tmp/PlayAround/.venv/lib/python3.6/site-packages
running install_egg_info
Removing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Writing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
COOL
它也适用于我以前的问题版本，它更复杂。
无论如何，感谢您的帮助:)
我有一个项目，我用 C (python c-api) 为它构建了一个 Class，我在 python 项目中扩展了更多。该项目的目的是为 C 库提供一个测试框架。主要针对C库的每个pull request执行测试
项目需要从Nexus服务器下载C库的相关build，编译依赖C库的python class，然后进行测试。
问题：编译C代码后导入/重新加载项目模块。
问题: 在我看来，在依赖于C库的每个函数中导入并不是那么优雅，所以我尝试调用reload，但似乎它不起作用，或者至少不像我期望的那样。
代码代码超级简化来说明问题，您可以查看此线程历史以查看以前的代码。
main.py
from utils.my_custom_py import MyCustomExtended
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
for module_name in list(sys.modules.keys()):
m = sys.modules.get(module_name)
if not hasattr(m, '__file__'):
continue
if getattr(m, '__name__', None) in [None, '__mp_main__', '__main__']:
continue
try:
# superreload(m) # from ==> IPython.extensions
# sys.modules[module_name] = reload(m)
reload(m)
except Exception as e:
...
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
utils.my_custom_py.py
from importlib.util import find_spec
if find_spec('custom2'):
import custom2
else:
class custom2:
class Custom:
...
class MyCustomExtended(custom2.Custom):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
setup.py
from distutils.core import Extension, setup
custom_ext = Extension("custom2", ["src/custom.c"])
setup(name="custom2", version="1.0", ext_modules=[custom_ext])
src.custom.c 取自：docs.python.org
错误输出:
running clean
running build
running build_ext
building 'custom2' extension
creating build
creating build/temp.linux-x86_64-3.6
creating build/temp.linux-x86_64-3.6/src
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/tmp/PlayAround/.venv/include -I/usr/include/python3.6m -c src/custom.c -o build/temp.linux-x86_64-3.6/src/custom.o
creating build/lib.linux-x86_64-3.6
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/src/custom.o -o build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so
running install
running install_lib
copying build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so -> /tmp/PlayAround/.venv/lib/python3.6/site-packages
running install_egg_info
Removing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Writing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Traceback (most recent call last):
File "./main.py", line 38, in <module>
main()
File "./main.py", line 33, in main
MyCustomExtended(1, 2, 3)
File "/tmp/PlayAround/utils/my_custom_py.py", line 12, in __init__
super().__init__(*args, **kwargs)
TypeError: object.__init__() takes no parameters
主要工作:
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
from utils.my_custom_py import MyCustomExtended
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
除了代码中的各种小问题外，您还试图重新加载所有存在的模块，同时默默地吞下错误。重新加载 utils 模块按预期工作：
import utils
from importlib.util import find_spec
from importlib import reload
from os import system, stat
import weakref
import sys
def setup():
if system('python3 setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def main():
if find_spec('custom2') is None:
setup()
for module_name in list(sys.modules.keys()):
m = sys.modules.get(module_name)
if not hasattr(m, '__file__'):
continue
if getattr(m, '__name__', None) in [None, '__mp_main__', '__main__']:
continue
reload(utils)
MyCustomExtended = utils.MyCustomExtended
print(MyCustomExtended)
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
我建议您只重新加载应用程序中的模块。
最后，我通过反复试验找到了解决方案。 对于此示例代码 id 执行了以下操作：
from utils.my_custom_py import MyCustomExtended
from importlib.util import find_spec
from importlib import reload
from sys import modules
from os import system
def setup():
if system('./setup.py clean build install') > 0:
raise SystemError("Failed to setup python c-api extention class")
def reload_my_libs():
global MyCustomExtended
reload(modules['utils'])
reload(modules['utils.my_custom_py'])
from utils.my_custom_py import MyCustomExtended
def main():
if find_spec('custom2') is None:
setup()
reload_my_libs()
MyCustomExtended(1, 2, 3)
print("COOL")
if __name__ == "__main__":
main()
结果，我得到了：
running clean
running build
running build_ext
building 'custom2' extension
creating build
creating build/temp.linux-x86_64-3.6
creating build/temp.linux-x86_64-3.6/src
x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/tmp/PlayAround/.venv/include -I/usr/include/python3.6m -c src/custom.c -o build/temp.linux-x86_64-3.6/src/custom.o
creating build/lib.linux-x86_64-3.6
x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/src/custom.o -o build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so
running install
running install_lib
copying build/lib.linux-x86_64-3.6/custom2.cpython-36m-x86_64-linux-gnu.so -> /tmp/PlayAround/.venv/lib/python3.6/site-packages
running install_egg_info
Removing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
Writing /tmp/PlayAround/.venv/lib/python3.6/site-packages/custom2-1.0.egg-info
COOL
它也适用于我以前的问题版本，它更复杂。
无论如何，感谢您的帮助:)