HDF5 消耗数据空间的所有资源 ID 并退出 (C++ API)

Question

编辑：事实证明，HDF5 C++ 库 1.8.15 中存在关于 DataSet.getSpace() 的（已知？）内存泄漏。

我编写了一个简单的解决方法函数，通过补救 C 库来避免泄漏代码。因此我将问题标记为已解决

H5::DataSpace getSpace( H5::DataSet& ds )
{
  hid_t id2 = ds.getId();
  hid_t myspace = H5Dget_space(id2);
  H5::DataSpace origspace( myspace );
  H5Sclose( myspace );
  return origspace;
}

这可能与 https://github.com/h5py/h5py/issues/480 有关，后者与 python 有关。

HDF5（C++？）API 中存在与内存 spaces 相关的大量内存泄漏。

编辑：找到 http://lists.hdfgroup.org/pipermail/hdf-forum_lists.hdfgroup.org/2015-August/008854.html

我在我的软件的典型运行期间多次 read/write HDF5 数据集。每次我读取行的一个子集时，我必须创建一个内存 space 和一个文件 space（使 hyperslab 成为 select 要读取的数据子集）。这些显然在 HDF5 的幕后消耗 "Resource IDs"。因为我做了很多很多次（因为我将行添加到矩阵并读取它们），所以最终这些运行出来了。即使我 "close" 数据 space 正确，也会发生这种情况。哎呀，如果我不创建新数据 space 而是通过 setExtentSimple().

重用数据 space，它甚至似乎会发生

这是重现问题的最小玩具代码！我什至根本不需要添加/read/write！只需创建一个 space 就好像我要阅读它然后 BAM 一样。哎呀，我只在数据 space 中创建了 1 行，就假装我会读一大堆。请注意，这会消耗大量内存（内存泄漏！）它并不是每次都释放数据空间，似乎......当函数中的 H5::DataSpace memspace 超出范围时，这是人们所期望的！！它退出时出现如下错误：

HDF5-DIAG: Error detected in HDF5 (1.8.15-patch1) thread 0:
  #000: H5S.c line 1393 in H5Screate_simple(): unable to register dataspace ID
    major: Object atom
    minor: Unable to register new atom
  #001: H5I.c line 902 in H5I_register(): no IDs available in type
    major: Object atom
    minor: Out of IDs for group
terminate called after throwing an instance of 'H5::DataSpaceIException'

编译：

    h5c++ -std=c++11 -O2 -g testhdf5id.cpp -o testhdf5id.exe

文件testhdf5id.cpp:

#include <H5Cpp.h>
#include <cstdlib>
#include <cstdio>
#include <iostream>
#include <string>
#include <vector>

#include <cstdint>

//Open a dataset. Write 1 row to it. Read every row one at a time. Do this a gazillion times.

//Because what the heck? Hopefully it won't cache it.


hsize_t currid;

void addrow( H5::DataSet& ds, const std::vector<double>& rowtowrite )
{
  //Get the space (since it may have grown in length since last time of course )
  H5::DataSpace origspace = ds.getSpace();

  //get the rank, even though I know it is 2
  int rank = origspace.getSimpleExtentNdims();

  //Get the actual dimensions of the ranks.
  hsize_t dims[rank];
  int ndims = origspace.getSimpleExtentDims( dims, NULL);

  //Want to ADD a row, so need to offset at row = nrows, and col = 0;
  hsize_t offset[rank] = { dims[0], 0 }; 
  hsize_t dims_toadd[rank] = { 1, rowtowrite.size() }; //will write 1 row, ncols columns.

  //Compute "new" size (extended by 1 row).
  hsize_t size[rank] = { dims[0]+dims_toadd[0], rowtowrite.size() };

  //Do the extension.
  ds.extend( size );

  //Get the new (extended) space, and select the hyperslab to write the row to.
  origspace = ds.getSpace();
  origspace.selectHyperslab( H5S_SELECT_SET, dims_toadd, offset );

  //Make the "memory" data space?
  H5::DataSpace toaddspace(rank, dims_toadd);

  ds.write(  rowtowrite.data(), H5::PredType::NATIVE_DOUBLE, toaddspace, origspace );

  //Can close toaddspace/origspace with no effect.
  //Can also close/open data set at the beginning of each time with no effect.
}

hsize_t getnrows( H5::DataSet& ds )
{
  H5::DataSpace origspace = ds.getSpace();
  int rank = origspace.getSimpleExtentNdims();
  hsize_t dims[rank];
  int ndims = origspace.getSimpleExtentDims( dims, NULL);
  hsize_t nrows=dims[0];
  hsize_t ncols=dims[1];

  return nrows;
}

std::vector<double> read1row( H5::DataSet& ds, const hsize_t& row )
{
  H5::DataSpace origspace = ds.getSpace();
  int rank = origspace.getSimpleExtentNdims();
  hsize_t dims[rank];
  int ndims = origspace.getSimpleExtentDims( dims, NULL);
  hsize_t nrows=dims[0];
  hsize_t ncols=dims[1];
  std::vector<double> returnvect( ncols );

  if(row >= nrows )
    {
      fprintf(stderr, "REV: ERROR, trying to read a row outside of matrix ([%lld] vs mat size [%lld])\n", row, nrows);
      exit(1);
    }

  hsize_t targrowoffset = row;
  hsize_t targcoloffset = 0;
  hsize_t dimsmem[rank] = {1,  ncols};
  H5::DataSpace memspace(rank, dimsmem);

  int id=memspace.getId();
  if(id==INT_MAX)
    {
      fprintf(stdout, "WOW, ID == INT_MAX, errortime?\n");
    }

  if( (row+1) % 100000 == 0 )
    {
      fprintf(stdout, "MEMSPACE ID [%d]\n", memspace.getId() );
    }

  hsize_t offset[rank] = { targrowoffset, targcoloffset };
  origspace.selectHyperslab( H5S_SELECT_SET, dimsmem, offset );
  ds.read( returnvect.data(), H5::PredType::NATIVE_DOUBLE, memspace, origspace );

  return returnvect;
}

void readallrows( H5::DataSet& ds )
{
  hsize_t nrows = getnrows( ds );
  //fprintf(stdout, "Read [%lld] rows\n", nrows );
  for(hsize_t r=0; r<nrows; ++r)
    {
      std::vector<double> gotvect = read1row( ds, r );
      if( gotvect[0] != (double)r )
    {
      fprintf(stderr, "ERROR IN READING...\n");
      exit(1);
    }
    }
  return;
}

std::vector<double> readlastrow( H5::DataSet& ds )
{
  H5::DataSpace origspace = ds.getSpace();
  int rank = origspace.getSimpleExtentNdims();
  hsize_t dims[rank];
  int ndims = origspace.getSimpleExtentDims( dims, NULL);
  hsize_t nrows=dims[0];
  hsize_t ncols=dims[1];
  std::vector<double> returnvect( ncols );



  hsize_t targrowoffset = nrows-1;
  hsize_t targcoloffset = 0;
  hsize_t dimsmem[rank] = {1,  ncols};
  H5::DataSpace memspace(rank, dimsmem);

  hsize_t offset[rank] = { targrowoffset, targcoloffset };
  origspace.selectHyperslab( H5S_SELECT_SET, dimsmem, offset );
  ds.read( returnvect.data(), H5::PredType::NATIVE_DOUBLE, memspace, origspace );

  return returnvect;
}


int fakereadlastrow( H5::DataSet& ds, const int& previd )
{
  H5::DataSpace origspace = ds.getSpace();
  int rank = origspace.getSimpleExtentNdims();
  hsize_t dims[rank];
  int ndims = origspace.getSimpleExtentDims( dims, NULL);
  hsize_t nrows=dims[0];
  hsize_t ncols=dims[1];
  std::vector<double> returnvect( ncols );



  hsize_t targrowoffset = nrows-1;
  hsize_t targcoloffset = 0;
  hsize_t dimsmem[rank] = {1,  ncols};
  H5::DataSpace memspace(rank, dimsmem);

  hsize_t offset[rank] = { targrowoffset, targcoloffset };
  origspace.selectHyperslab( H5S_SELECT_SET, dimsmem, offset );

  //REV: Would read here, but I don't for speed.
  //ds.read( returnvect.data(), H5::PredType::NATIVE_DOUBLE, memspace, origspace );
  int id =   memspace.getId();

  if(id % 1000000 == 0 )
    {
      fprintf(stdout, "PREV ID: [%d] now ID: [%d]\n", previd, id);
    }
  return id;
  //return returnvect;
}

int main()
{
  std::string fname = "testhdf5file.h5";
  H5::H5File f( fname, H5F_ACC_TRUNC );
  std::string dsetname = "dset1";

  const hsize_t nranks = 2;
  const hsize_t ncols = 20;
  const hsize_t nrows = 0; //start with zero rows.

  hsize_t  dims[nranks] = {nrows, ncols};
  hsize_t  max_dims[nranks] = {H5S_UNLIMITED, ncols};

  H5::DataSpace dataspace( nranks, dims, max_dims );
  H5::DSetCreatPropList prop; //could set properties, but whatever.

  const hsize_t nrows_chunk = 100; //Need to mess with CACHE size too!
  hsize_t  chunk_dims[nranks] = { nrows_chunk, ncols};
  prop.setChunk(nranks, chunk_dims);


  //Create the dataset 
  H5::DataSet ds = f.createDataSet( dsetname, H5::PredType::NATIVE_DOUBLE,
                    dataspace, prop);


  size_t nrowstoadd=1;

  for(size_t t=0; t<nrowstoadd; ++t)
    {
      std::vector<double> rowtowrite( ncols, (double)t );
      addrow( ds, rowtowrite );
    }

  int id=0;
  for(size_t t=0; t<10000000; ++t)
    {

      //readallrows( ds );

      //readlastrow();

      id = fakereadlastrow(ds, id);


      //f.flush(H5F_SCOPE_GLOBAL);
    }

  return 0;
}

Answer 1

编辑：正如编辑问题中提到的，这是由于 H5::DataSet::getSpace() 中的大量内存泄漏造成的。这是解决问题的解决方法代码（将对 dataset.getSpace() 的所有调用替换为 getSpace(dataset);

H5::DataSpace getSpace( H5::DataSet& ds )
{
  hid_t id2 = ds.getId();
  hid_t myspace = H5Dget_space(id2);
  H5::DataSpace origspace( myspace );
  H5Sclose( myspace );
  return origspace;
}

HDF5 消耗数据空间的所有资源 ID 并退出 (C++ API)

HDF5 consumes all resource IDs for DataSpaces and exits (C++ API)

c++

hdf5