Python decompression relative performance?

Question

TLDR; Of the various compression algorithms available in python gzip, bz2, lzma, etc, which has the best decompression performance? Full discussion: Python 3 has various modules for compressing/decompressing data including gzip, bz2 and lzma. gzip and bz2 additionally have different compression levels you can set. If my goal is to balance file size (/compression ratio) and decompression speed (compression speed

Accepted Answer

You can use Python-bloscIt is very fast and for small arrays (<2GB) also quite easy to use. On easily compressable data like your example, it is often faster to compress the data for IO operations. (SATA-SSD: about 500 MB/s, PCIe- SSD: up to 3500MB/s) In the decompression step the array allocation is the most costly part. If your images are of similar shape you can avoid repeated memory allocation.ExampleA contigous array is assumed for the following example.import bloscimport pickledef compress(arr,Path):    #c = blosc.compress_ptr(arr.__array_interface__['data'][0], arr.size, arr.dtype.itemsize, clevel=3,cname='lz4',shuffle=blosc.SHUFFLE)    c = blosc.compress_ptr(arr.__array_interface__['data'][0], arr.size, arr.dtype.itemsize, clevel=3,cname='zstd',shuffle=blosc.SHUFFLE)    f=open(Path,"wb")    pickle.dump((arr.shape, arr.dtype),f)    f.write(c)    f.close()    return c,arr.shape, arr.dtypedef decompress(Path):    f=open(Path,"rb")    shape,dtype=pickle.load(f)    c=f.read()    #array allocation takes most of the time    arr=np.empty(shape,dtype)    blosc.decompress_ptr(c, arr.__array_interface__['data'][0])    return arr#Pass a preallocated array if you have many similar imagesdef decompress_pre(Path,arr):    f=open(Path,"rb")    shape,dtype=pickle.load(f)    c=f.read()    #array allocation takes most of the time    blosc.decompress_ptr(c, arr.__array_interface__['data'][0])    return arrBenchmarks#blosc.SHUFFLE, cname='zstd' -> 4728KB,  %timeit compress(arr,"Test.dat")1.03 s ± 12.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)#611 MB/s%timeit decompress("Test.dat")146 ms ± 481 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)#4310 MB/s%timeit decompress_pre("Test.dat",arr)50.9 ms ± 438 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)#12362 MB/s#blosc.SHUFFLE, cname='lz4' -> 9118KB, %timeit compress(arr,"Test.dat")32.1 ms ± 437 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)#19602 MB/s%timeit decompress("Test.dat")146 ms ± 332 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)#4310 MB/s%timeit decompress_pre("Test.dat",arr)53.6 ms ± 82.9 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)#11740 MB/sEditThis version is more for general use. It does handle f-contiguous, c-contiguous and non-contiguous arrays and arrays >2GB. Also have a look at bloscpack.import bloscimport pickledef compress(file, arr,clevel=3,cname='lz4',shuffle=1):    """    file           path to file    arr            numpy nd-array    clevel         0..9    cname          blosclz,lz4,lz4hc,snappy,zlib    shuffle        0-> no shuffle, 1->shuffle,2->bitshuffle    """    max_blk_size=100_000_000 #100 MB     shape=arr.shape    #dtype np.object is not implemented    if arr.dtype==np.object:        raise(TypeError("dtype np.object is not implemented"))    #Handling of fortran ordered arrays (avoid copy)    is_f_contiguous=False    if arr.flags['F_CONTIGUOUS']==True:        is_f_contiguous=True        arr=arr.T.reshape(-1)    else:        arr=np.ascontiguousarray(arr.reshape(-1))    #Writing    max_num=max_blk_size//arr.dtype.itemsize    num_chunks=arr.size//max_num    if arr.size%max_num!=0:        num_chunks+=1    f=open(file,"wb")    pickle.dump((shape,arr.size,arr.dtype,is_f_contiguous,num_chunks,max_num),f)    size=np.empty(1,np.uint32)    num_write=max_num    for i in range(num_chunks):        if max_num*(i+1)>arr.size:            num_write=arr.size-max_num*i        c = blosc.compress_ptr(arr[max_num*i:].__array_interface__['data'][0], num_write,                                arr.dtype.itemsize, clevel=clevel,cname=cname,shuffle=shuffle)        size[0]=len(c)        size.tofile(f)        f.write(c)    f.close()def decompress(file,prealloc_arr=None):    f=open(file,"rb")    shape,arr_size,dtype,is_f_contiguous,num_chunks,max_num=pickle.load(f)    if prealloc_arr is None:        if prealloc_arr.flags['F_CONTIGUOUS']==True            prealloc_arr=prealloc_arr.T        if prealloc_arr.flags['C_CONTIGUOUS']!=True            raise(TypeError("Contiguous array is needed"))        arr=np.empty(arr_size,dtype)    else:        arr=np.frombuffer(prealloc_arr.data, dtype=dtype, count=arr_size)    for i in range(num_chunks):        size=np.fromfile(f,np.uint32,count=1)        c=f.read(size[0])        blosc.decompress_ptr(c, arr[max_num*i:].__array_interface__['data'][0])    f.close()    #reshape    if is_f_contiguous:        arr=arr.reshape(shape[::-1]).T    else:        arr=arr.reshape(shape)    return arr

Advertisement

Answer

You can use Python-blosc