如何 compress/decompress 使用 Fast-LZMA2 缓冲

Question

我想 compress/decompress 使用 7Zip 的 fast-LZMA2 的无符号字符缓冲区：https://github.com/conor42/fast-lzma2

示例中有两个函数：


static int compress_file(FL2_CStream *fcs)
{
    unsigned char in_buffer[8 * 1024];
    unsigned char out_buffer[4 * 1024];
    FL2_inBuffer in_buf = { in_buffer, sizeof(in_buffer), sizeof(in_buffer) };
    FL2_outBuffer out_buf = { out_buffer, sizeof(out_buffer), 0 };
    size_t res = 0;
    size_t in_size = 0;
    size_t out_size = 0;
    do {
        if (in_buf.pos == in_buf.size) {
            in_buf.size = fread(in_buffer, 1, sizeof(in_buffer), fin);
            in_size += in_buf.size;
            in_buf.pos = 0;
        }
        res = FL2_compressStream(fcs, &out_buf, &in_buf);
        if (FL2_isError(res))
            goto error_out;

        fwrite(out_buf.dst, 1, out_buf.pos, fout);
        out_size += out_buf.pos;
        out_buf.pos = 0;

    } while (in_buf.size == sizeof(in_buffer));
    do {
        res = FL2_endStream(fcs, &out_buf);
        if (FL2_isError(res))
            goto error_out;

        fwrite(out_buf.dst, 1, out_buf.pos, fout);
        out_size += out_buf.pos;
        out_buf.pos = 0;
    } while (res);
    fprintf(stdout, "\t%ld -> %ld\n", in_size, out_size);
    
    return 0;

error_out:
    fprintf(stderr, "Error: %s\n", FL2_getErrorName(res));
    return 1;
}
static int decompress_file(FL2_DStream *fds)
{
    unsigned char in_buffer[4 * 1024];
    unsigned char out_buffer[8 * 1024];
    FL2_inBuffer in_buf = { in_buffer, sizeof(in_buffer), sizeof(in_buffer) };
    FL2_outBuffer out_buf = { out_buffer, sizeof(out_buffer), 0 };
    size_t res;
    size_t in_size = 0;
    size_t out_size = 0;
    do {
        if (in_buf.pos == in_buf.size) {
            in_buf.size = fread(in_buffer, 1, sizeof(in_buffer), fout);
            in_size += in_buf.size;
            in_buf.pos = 0;
        }
        res = FL2_decompressStream(fds, &out_buf, &in_buf);
        if (FL2_isError(res))
            goto error_out;
        /* Discard the output. XXhash will verify the integrity. */
        out_size += out_buf.pos;
        out_buf.pos = 0;
    } while (res && in_buf.size);
    
    fprintf(stdout, "\t%ld -> %ld\n", in_size, out_size);
    
    return 0;

error_out:
    fprintf(stderr, "Error: %s\n", FL2_getErrorName(res));
    return 1;
}

但我不知道如何让它与缓冲区一起工作并且没有像 8*1024 这样的大小限制像 zlib deflate 压缩。

我想要类似的东西 LZMA2_Compress(void* buffer,size_t bufferSize);

和LZMA2_Decompress(void* buffer,size_t bufferSize);

我想对一些大文件使用此算法，Fast LZMA2 是我发现的最快的高比率压缩，请不要建议我使用其他方法。

这是我的测试代码，它可以工作，但只需要更正信息： https://gist.github.com/Bit00009/3241bb66301f8aaba16074537d094e61

Answer 1

检查所有可用函数的头文件。这个看起来像你需要的那个。您需要将缓冲区转换为 (void *).

高级函数

快-lzma2.h

...
/*! FL2_compress() :
 *  Compresses `src` content as a single LZMA2 compressed stream into already allocated `dst`.
 *  Call FL2_compressMt() to use > 1 thread. Specify nbThreads = 0 to use all cores.
 *  @return : compressed size written into `dst` (<= `dstCapacity),
 *            or an error code if it fails (which can be tested using FL2_isError()). */
FL2LIB_API size_t FL2LIB_CALL FL2_compress(void* dst, size_t dstCapacity,
    const void* src, size_t srcSize,
    int compressionLevel);
...

内存和选项的管理

要进行显式内存管理（设置字典大小、缓冲区大小等），您需要创建上下文：

快-lzma2.h

/*= Compression context
 *  When compressing many times, it is recommended to allocate a context just once,
 *  and re-use it for each successive compression operation. This will make workload
 *  friendlier for system's memory. The context may not use the number of threads requested
 *  if the library is compiled for single-threaded compression or nbThreads > FL2_MAXTHREADS.
 *  Call FL2_getCCtxThreadCount to obtain the actual number allocated. */
typedef struct FL2_CCtx_s FL2_CCtx;
FL2LIB_API FL2_CCtx* FL2LIB_CALL FL2_createCCtx(void);

比您可以使用 FL2_CCtx_setParameter() 在上下文中设置参数。 FL2_cParameter 中列出了参数的可能值，值 FL2_p_dictionarySize 将允许您设置字典大小。

/*! FL2_CCtx_setParameter() :
 *  Set one compression parameter, selected by enum FL2_cParameter.
 *  @result : informational value (typically, the one being set, possibly corrected),
 *            or an error code (which can be tested with FL2_isError()). */
FL2LIB_API size_t FL2LIB_CALL FL2_CCtx_setParameter(FL2_CCtx* cctx, FL2_cParameter param, size_t value);

最后你可以通过调用 FL2_compressCCtx()

来压缩缓冲区

/*! FL2_compressCCtx() :
 *  Same as FL2_compress(), but requires an allocated FL2_CCtx (see FL2_createCCtx()). */
FL2LIB_API size_t FL2LIB_CALL FL2_compressCCtx(FL2_CCtx* cctx,
    void* dst, size_t dstCapacity,
    const void* src, size_t srcSize,
    int compressionLevel);

如何 compress/decompress 使用 Fast-LZMA2 缓冲

How to compress/decompress buffer using Fast-LZMA2

c

c++

7zip

lzma

高级函数

内存和选项的管理