PyPI 打包、命名空间包和分包问题

Question

我已将名为 ofunctions 的个人通用函数上传到 github，以便在我的项目之间共享它们，并进行单独的 CI 和覆盖率测试。 Link 到 github 项目 here。

到目前为止一切顺利，我有一个名为 ofunctions 的包，其中包含多个子包，例如 ofunctions.network。

我希望能够安装子包而不必安装整个包，即 pip install ofunctions.network。所以我创建了一个 setup.py 文件来创建必要的 dist 文件以上传到 PyPI。

我的问题：

每当我使用 python setup.py sdist bdist_wheel 时，它都会生成完整的 ofunctions 包和每个子包的包，但是：

像 ofunctions.network-0.5.0.tar.gz 这样的源包只包含子包（预期行为）
wheel packages like ofunctions.network-0.5.0-py3-non-any.whl which contain the whole package (unexpected behavior)

wheel 包包含整个 ofunctions 库，包括所有子包，显然应该只包含与源 dist 文件相同的子包。

谁能看看我的 setup.py 文件并告诉我为什么 sdist 和 wheel 文件不只包含完全相同的子包？

#! /usr/bin/env python
#  -*- coding: utf-8 -*-
#
# This file is part of ofunctions package

"""
Namespace packaging here

# Make sure we declare an __init__.py file as namespace holder in the package root containing the following

try:
    __import__('pkg_resources').declare_namespace(__name__)
except ImportError:
    from pkgutil import extend_path
    __path__ = extend_path(__path__, __name__)
"""

import codecs
import os

import pkg_resources
import setuptools


def get_metadata(package_file):
    """
    Read metadata from pacakge file
    """

    def _read(_package_file):
        here = os.path.abspath(os.path.dirname(__file__))
        with codecs.open(os.path.join(here, _package_file), 'r') as fp:
            return fp.read()

    _metadata = {}

    for line in _read(package_file).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            _metadata['version'] = line.split(delim)[1]
        if line.startswith('__description__'):
            delim = '"' if '"' in line else "'"
            _metadata['description'] = line.split(delim)[1]
    return _metadata


def parse_requirements(filename):
    """
    There is a parse_requirements function in pip but it keeps changing import path
    Let's build a simple one
    """
    try:
        with open(filename, 'r') as requirements_txt:
            install_requires = [
                str(requirement)
                for requirement
                in pkg_resources.parse_requirements(requirements_txt)
            ]
        return install_requires
    except OSError:
        print('WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'
              .format(filename))


def get_long_description(filename):
    with open(filename, 'r', encoding='utf-8') as readme_file:
        _long_description = readme_file.read()
    return _long_description


#  ######### ACTUAL SCRIPT ENTRY POINT

NAMESPACE_PACKAGE_NAME = 'ofunctions'
namespace_package_path = os.path.abspath(NAMESPACE_PACKAGE_NAME)
namespace_package_file = os.path.join(namespace_package_path, '__init__.py')
metadata = get_metadata(namespace_package_file)
requirements = parse_requirements(os.path.join(namespace_package_path, 'requirements.txt'))

# Generic namespace package
setuptools.setup(
    name=NAMESPACE_PACKAGE_NAME,
    namespace_packages=[NAMESPACE_PACKAGE_NAME],
    packages=setuptools.find_namespace_packages(include=['ofunctions.*']),
    version=metadata['version'],
    install_requires=requirements,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Topic :: Software Development",
        "Topic :: System",
        "Topic :: System :: Operating System",
        "Topic :: System :: Shells",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: Implementation :: CPython",
        "Programming Language :: Python :: Implementation :: PyPy",
        "Operating System :: POSIX :: Linux",
        "Operating System :: POSIX :: BSD :: FreeBSD",
        "Operating System :: POSIX :: BSD :: NetBSD",
        "Operating System :: POSIX :: BSD :: OpenBSD",
        "Operating System :: Microsoft",
        "Operating System :: Microsoft :: Windows",
        "License :: OSI Approved :: BSD License",
    ],
    description=metadata['description'],
    author='NetInvent - Orsiris de Jong',
    author_email='contact@netinvent.fr',
    url='https://github.com/netinvent/ofunctions',
    keywords=['network', 'bisection', 'logging'],
    long_description=get_long_description('README.md'),
    long_description_content_type="text/markdown",
    python_requires='>=3.5',
    # namespace packages don't work well with zipped eggs
    # ref https://packaging.python.org/guides/packaging-namespace-packages/
    zip_safe=False
)

for package in setuptools.find_namespace_packages(include=['ofunctions.*']):
    package_path = os.path.abspath(package.replace('.', os.sep))
    package_file = os.path.join(package_path, '__init__.py')
    metadata = get_metadata(package_file)
    requirements = parse_requirements(os.path.join(package_path, 'requirements.txt'))
    print(package_path)
    print(package_file)
    print(metadata)
    print(requirements)

    setuptools.setup(
        name=package,
        namespace_packages=[NAMESPACE_PACKAGE_NAME],
        packages=[package],
        package_data={package: ['__init__.py']},
        version=metadata['version'],
        install_requires=requirements,
        classifiers=[
            "Development Status :: 5 - Production/Stable",
            "Intended Audience :: Developers",
            "Topic :: Software Development",
            "Topic :: System",
            "Topic :: System :: Operating System",
            "Topic :: System :: Shells",
            "Programming Language :: Python",
            "Programming Language :: Python :: 3",
            "Programming Language :: Python :: Implementation :: CPython",
            "Programming Language :: Python :: Implementation :: PyPy",
            "Operating System :: POSIX :: Linux",
            "Operating System :: POSIX :: BSD :: FreeBSD",
            "Operating System :: POSIX :: BSD :: NetBSD",
            "Operating System :: POSIX :: BSD :: OpenBSD",
            "Operating System :: Microsoft",
            "Operating System :: Microsoft :: Windows",
            "License :: OSI Approved :: BSD License",
        ],
        description=metadata['description'],
        author='NetInvent - Orsiris de Jong',
        author_email='contact@netinvent.fr',
        url='https://github.com/netinvent/ofunctions',
        keywords=['network', 'bisection', 'logging'],
        long_description=get_long_description('README.md'),
        long_description_content_type="text/markdown",
        python_requires='>=3.5',
        # namespace packages don't work well with zipped eggs
        # ref https://packaging.python.org/guides/packaging-namespace-packages/
        zip_safe=False
    )

谢谢 8-|

Answer 1

好的，我想我找到了问题所在。 build 目录在 setuptools 运行s 之间没有清理。

更糟糕的是，构建目录 永远不会 清理，除非您手动删除它，所以旧的构建文件可能最终会出现在较新的 wheel 包构建中，我认为即使是在单个包构建中也是如此。

我在运行ning setuptools.setup() 运行之前添加了一个函数 clear_package_build_path()，它只清理 build/lib/package 目录。现在我的 wheel 文件只用必要的文件构建，不再膨胀。

例如这里是完整的工作代码：

#! /usr/bin/env python
#  -*- coding: utf-8 -*-
#
# This file is part of ofunctions package

"""
Namespace packaging here

# Make sure we declare an __init__.py file as namespace holder in the package root containing the following

try:
    __import__('pkg_resources').declare_namespace(__name__)
except ImportError:
    from pkgutil import extend_path
    __path__ = extend_path(__path__, __name__)
"""

import codecs
import os
import shutil

import pkg_resources
import setuptools


def get_metadata(package_file):
    """
    Read metadata from package file
    """

    def _read(_package_file):
        here = os.path.abspath(os.path.dirname(__file__))
        with codecs.open(os.path.join(here, _package_file), 'r') as fp:
            return fp.read()

    _metadata = {}

    for line in _read(package_file).splitlines():
        if line.startswith('__version__'):
            delim = '"' if '"' in line else "'"
            _metadata['version'] = line.split(delim)[1]
        if line.startswith('__description__'):
            delim = '"' if '"' in line else "'"
            _metadata['description'] = line.split(delim)[1]
    return _metadata


def parse_requirements(filename):
    """
    There is a parse_requirements function in pip but it keeps changing import path
    Let's build a simple one
    """
    try:
        with open(filename, 'r') as requirements_txt:
            install_requires = [
                str(requirement)
                for requirement
                in pkg_resources.parse_requirements(requirements_txt)
            ]
        return install_requires
    except OSError:
        print('WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'
              .format(filename))


def get_long_description(filename):
    with open(filename, 'r', encoding='utf-8') as readme_file:
        _long_description = readme_file.read()
    return _long_description


def clear_package_build_path(package_rel_path):
    """
    We need to clean build path, but setuptools will wait for build/lib/package_name so we need to create that
    """
    build_path = os.path.abspath(os.path.join('build', 'lib', package_rel_path))
    try:
        # We need to use shutil.rmtree() instead of os.remove() since the latter implementation
        # produces "WindowsError: [Error 5] Access is denied"
        shutil.rmtree('build')
    except FileNotFoundError:
        print('build path: {} does not exist'.format(build_path))
    # Now we need to create the 'build/lib/package/subpackage' path so setuptools won't fail
    os.makedirs(build_path)


#  ######### ACTUAL SCRIPT ENTRY POINT

NAMESPACE_PACKAGE_NAME = 'ofunctions'
namespace_package_path = os.path.abspath(NAMESPACE_PACKAGE_NAME)
namespace_package_file = os.path.join(namespace_package_path, '__init__.py')
metadata = get_metadata(namespace_package_file)
requirements = parse_requirements(os.path.join(namespace_package_path, 'requirements.txt'))

# First lets make sure build path is clean (avoiding namespace package pollution in subpackages)
# Clean build dir before every run so we don't make cumulative wheel files
clear_package_build_path(NAMESPACE_PACKAGE_NAME)

# Generic namespace package
setuptools.setup(
    name=NAMESPACE_PACKAGE_NAME,
    namespace_packages=[NAMESPACE_PACKAGE_NAME],
    packages=setuptools.find_namespace_packages(include=['ofunctions.*']),
    version=metadata['version'],
    install_requires=requirements,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Topic :: Software Development",
        "Topic :: System",
        "Topic :: System :: Operating System",
        "Topic :: System :: Shells",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: Implementation :: CPython",
        "Programming Language :: Python :: Implementation :: PyPy",
        "Operating System :: POSIX :: Linux",
        "Operating System :: POSIX :: BSD :: FreeBSD",
        "Operating System :: POSIX :: BSD :: NetBSD",
        "Operating System :: POSIX :: BSD :: OpenBSD",
        "Operating System :: Microsoft",
        "Operating System :: Microsoft :: Windows",
        "License :: OSI Approved :: BSD License",
    ],
    description=metadata['description'],
    author='NetInvent - Orsiris de Jong',
    author_email='contact@netinvent.fr',
    url='https://github.com/netinvent/ofunctions',
    keywords=['network', 'bisection', 'logging'],
    long_description=get_long_description('README.md'),
    long_description_content_type="text/markdown",
    python_requires='>=3.5',
    # namespace packages don't work well with zipped eggs
    # ref https://packaging.python.org/guides/packaging-namespace-packages/
    zip_safe=False
)



for package in setuptools.find_namespace_packages(include=['ofunctions.*']):
    rel_package_path = package.replace('.', os.sep)
    package_path = os.path.abspath(rel_package_path)
    package_file = os.path.join(package_path, '__init__.py')
    metadata = get_metadata(package_file)
    requirements = parse_requirements(os.path.join(package_path, 'requirements.txt'))
    print(package_path)
    print(package_file)
    print(metadata)
    print(requirements)

    # Again, we need to clean build paths between runs
    clear_package_build_path(rel_package_path)

    setuptools.setup(
        name=package,
        namespace_packages=[NAMESPACE_PACKAGE_NAME],
        packages=[package],
        package_data={package: ['__init__.py']},
        version=metadata['version'],
        install_requires=requirements,
        classifiers=[
            "Development Status :: 5 - Production/Stable",
            "Intended Audience :: Developers",
            "Topic :: Software Development",
            "Topic :: System",
            "Topic :: System :: Operating System",
            "Topic :: System :: Shells",
            "Programming Language :: Python",
            "Programming Language :: Python :: 3",
            "Programming Language :: Python :: Implementation :: CPython",
            "Programming Language :: Python :: Implementation :: PyPy",
            "Operating System :: POSIX :: Linux",
            "Operating System :: POSIX :: BSD :: FreeBSD",
            "Operating System :: POSIX :: BSD :: NetBSD",
            "Operating System :: POSIX :: BSD :: OpenBSD",
            "Operating System :: Microsoft",
            "Operating System :: Microsoft :: Windows",
            "License :: OSI Approved :: BSD License",
        ],
        description=metadata['description'],
        author='NetInvent - Orsiris de Jong',
        author_email='contact@netinvent.fr',
        url='https://github.com/netinvent/ofunctions',
        keywords=['network', 'bisection', 'logging'],
        long_description=get_long_description('README.md'),
        long_description_content_type="text/markdown",
        python_requires='>=3.5',
        # namespace packages don't work well with zipped eggs
        # ref https://packaging.python.org/guides/packaging-namespace-packages/
        zip_safe=False
    )

作为侧节点，我注意到 os.remove() 会不时以 WindowsError: [Error 5] Access is denied 失败，因为 os.remove() 等待所有句柄关闭，这可能需要时间，因为垃圾收集器（AFAIK）。使用 shutil.rmtree() 在任何情况下都有效。

PyPI 打包、命名空间包和分包问题

PyPI packaging, namespace packages and subpackaging problems

setuptools

setup.py

namespace-package

python-packaging