compress.zstd #

Description

compress.zstd is a module that assists in the compression and decompression of binary data using zstd compression.

fn check_zstd(code usize) !

check_zstd check the zstd error code, and return a error string.

fn compress(data []u8, params CompressParams) ![]u8

compresses an array of bytes using zstd and returns the compressed bytes in a new array extra compression parameters can be set by params

compressed := zstd.compress(b)!

fn decompress(data []u8, params DecompressParams) ![]u8

decompresses an array of bytes using zstd and returns the decompressed bytes in a new array extra decompression parameters can be set by params

decompressed := zstd.decompress(b)!

fn default_c_level() int

default_c_level return default compression level.

fn get_error_name(code usize) string

get_error_name provides readable string from an error code.

fn is_error(code usize) bool

is_error tells if a usize function result is an error code.

fn load_array[T](fname string, params DecompressParams) ![]T

load_array return an array which data is decompressed from a file fname. extra decompression parameters can be set by params

fn max_c_level() int

max_c_level return maximum compression level available.

fn min_c_level() int

min_c_level return minimum negative compression level allowed.

fn new_cctx(params CompressParams) !&ZSTD_CCtx

new_cctx create a compression context extra compression parameters can be set by params

fn new_dctx(params DecompressParams) !&ZSTD_DCtx

new_dctx create a decompression context extra decompression parameters can be set by params

fn store_array[T](fname string, array []T, params CompressParams) !

store_array compress an array's data, and store it to file fname. extra compression parameters can be set by params WARNING: Because struct padding, some data in struct may be marked unused. So, when store_array, it will cause memory fsanitize fail with 'use-of-uninitialized-value'. It can be safely ignored. For example, following struct may cause memory fsanitize fail: struct MemoryTrace { operation u8 address u64 size u8 } By changing it into following , you can pass the memory fsanitize check : struct MemoryTrace { operation u64 address u64 size u64 }

fn version_number() u32

version_number return runtime library version, the value is (MAJOR100100 + MINOR*100 + RELEASE).

fn version_string() string

version_string return runtime library version, like "1.5.5".

type ZSTD_CCtx = C.ZSTD_CCtx

ZSTD_CCtx zstd compression context struct

fn (mut u ZSTD_CCtx) set_parameter(c_param ZSTD_cParameter, val int) usize

set_parameter set compression parameter c_param to value val

fn (mut u ZSTD_CCtx) compress_stream2(output &ZSTD_outBuffer, input &ZSTD_inBuffer, mode ZSTD_EndDirective) usize

compress_stream2 do stream compress on input, and store compressed data in output. mode: .zstd_e_continue => continue stream compression. .zstd_e_flush => flush data .zstd_e_end => it is the last frame

fn (mut u ZSTD_CCtx) free_cctx() usize

free_cctx free a compression context

type ZSTD_DCtx = C.ZSTD_DCtx

ZSTD_DCtx zstd decompression context struct

fn (mut u ZSTD_DCtx) set_parameter(d_param ZSTD_dParameter, val int) usize

set_parameter set decompression parameter d_param to value val

fn (mut u ZSTD_DCtx) decompress_stream(output &ZSTD_outBuffer, input &ZSTD_inBuffer) usize

decompress_stream do stream decompress on input, and store decompressed data in output. return remaining bytes in input stream

fn (mut u ZSTD_DCtx) free_dctx() usize

free_cctx free a compression context

enum ZSTD_EndDirective {
	// collect more data, encoder decides when to output compressed result, for optimal compression ratio
	zstd_e_continue = 0
	// flush any data provided so far,
	// it creates (at least) one new block, that can be decoded immediately on reception;
	// frame will continue: any future data can still reference previously compressed data, improving compression.
	// note : multithreaded compression will block to flush as much output as possible.
	zstd_e_flush = 1
	// flush any remaining data _and_ close current frame.
	// note that frame is only closed after compressed data is fully flushed (return value == 0).
	// After that point, any additional data starts a new frame.
	// note : each frame is independent (does not reference any content from previous frame).
	// note : multithreaded compression will block to flush as much output as possible.
	zstd_e_end = 2
}

enum ZSTD_ResetDirective {
	zstd_reset_session_only           = 1
	zstd_reset_parameters             = 2
	zstd_reset_session_and_parameters = 3
}

enum ZSTD_cParameter {
	// compression parameters
	// Note: When compressing with a ZSTD_CDict these parameters are superseded
	// by the parameters used to construct the ZSTD_CDict.
	// See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict).
	//
	// Set compression parameters according to pre-defined cLevel table.
	// Note that exact compression parameters are dynamically determined,
	// depending on both compression level and srcSize (when known).
	// Default level is ZSTD_CLEVEL_DEFAULT==3.
	// Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT.
	// Note 1 : it's possible to pass a negative compression level.
	// Note 2 : setting a level does not automatically set all other compression parameters
	// to default. Setting this will however eventually dynamically impact the compression
	// parameters which have not been manually set. The manually set
	// ones will 'stick'.
	zstd_c_compression_level = 100
	// Advanced compression parameters :
	// It's possible to pin down compression parameters to some specific values.
	// In which case, these values are no longer dynamically selected by the compressor
	//
	// Maximum allowed back-reference distance, expressed as power of 2.
	// This will set a memory budget for streaming decompression,
	// with larger values requiring more memory
	// and typically compressing more.
	// Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX.
	// Special: value 0 means "use default windowLog".
	// Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT
	// requires explicitly allowing such size at streaming decompression stage.
	zstd_c_window_log = 101
	// Size of the initial probe table, as a power of 2.
	// Resulting memory usage is (1 << (hashLog+2)).
	// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX.
	// Larger tables improve compression ratio of strategies <= dFast,
	// and improve speed of strategies > dFast.
	// Special: value 0 means "use default hashLog".
	zstd_c_hash_log = 102
	// Size of the multi-probe search table, as a power of 2.
	// Resulting memory usage is (1 << (chainLog+2)).
	// Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX.
	// Larger tables result in better and slower compression.
	// This parameter is useless for "fast" strategy.
	// It's still useful when using "dfast" strategy,
	// in which case it defines a secondary probe table.
	// Special: value 0 means "use default chainLog".
	zstd_c_chain_log = 103
	// Number of search attempts, as a power of 2.
	// More attempts result in better and slower compression.
	// This parameter is useless for "fast" and "dFast" strategies.
	// Special: value 0 means "use default searchLog".
	zstd_c_search_log = 104
	// Minimum size of searched matches.
	// Note that Zstandard can still find matches of smaller size,
	// it just tweaks its search algorithm to look for this size and larger.
	// Larger values increase compression and decompression speed, but decrease ratio.
	// Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX.
	// Note that currently, for all strategies < btopt, effective minimum is 4.
	// , for all strategies > fast, effective maximum is 6.
	// Special: value 0 means "use default minMatchLength".
	zstd_c_min_match = 105
	// Impact of this field depends on strategy.
	// For strategies btopt, btultra & btultra2:
	// Length of Match considered "good enough" to stop search.
	// Larger values make compression stronger, and slower.
	// For strategy fast:
	// Distance between match sampling.
	// Larger values make compression faster, and weaker.
	// Special: value 0 means "use default targetLength".
	zstd_c_target_length = 106
	// See ZSTD_strategy enum definition.
	// The higher the value of selected strategy, the more complex it is,
	// resulting in stronger and slower compression.
	// Special: value 0 means "use default strategy".
	zstd_c_strategy = 107
	// LDM mode parameters
	// Enable long distance matching.
	// This parameter is designed to improve compression ratio
	// for large inputs, by finding large matches at long distance.
	// It increases memory usage and window size.
	// Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB
	// except when expressly set to a different value.
	// Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and
	// compression strategy >= ZSTD_btopt (== compression level 16+)
	zstd_c_enable_long_distance_matching = 160
	// Size of the table for long distance matching, as a power of 2.
	// Larger values increase memory usage and compression ratio,
	// but decrease compression speed.
	// Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX
	// default: windowlog - 7.
	// Special: value 0 means "automatically determine hashlog".
	zstd_c_ldm_hash_log = 161
	// Minimum match size for long distance matcher.
	// Larger/too small values usually decrease compression ratio.
	// Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX.
	// Special: value 0 means "use default value" (default: 64).
	zstd_c_ldm_min_match = 162
	// log size of each bucket in the ldm hash table for collision resolution.
	// Larger values improve collision resolution but decrease compression speed.
	// The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX.
	// Special: value 0 means "use default value" (default: 3).
	zstd_c_ldm_bucket_size_log = 163
	// Frequency of inserting/looking up entries into the LDM hash table.
	// Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN).
	// Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage.
	// Larger values improve compression speed.
	// Deviating far from default value will likely result in a compression ratio decrease.
	// Special: value 0 means "automatically determine hashRateLog".
	zstd_c_ldm_hash_rate_log = 164
	// frame parameters
	// Content size will be written into frame header _whenever known_ (default:1)
	// Content size must be known at the beginning of compression.
	// This is automatically the case when using ZSTD_compress2(),
	// For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize()
	zstd_c_content_size_flag = 200
	// A 32-bits checksum of content is written at end of frame (default:0)
	zstd_c_checksum_flag = 201
	// When applicable, dictionary's ID is written into frame header (default:1)
	zstd_c_dict_id_flag = 202
	// multi-threading parameters
	// These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD).
	// Otherwise, trying to set any other value than default (0) will be a no-op and return an error.
	// In a situation where it's unknown if the linked library supports multi-threading or not,
	// setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property.
	//
	// Select how many threads will be spawned to compress in parallel.
	// When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() :
	// ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller,
	// while compression is performed in parallel, within worker thread(s).
	// (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end :
	// in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call).
	// More workers improve speed, but also increase memory usage.
	// Default value is `0`, aka "single-threaded mode" : no worker is spawned,
	// compression is performed inside Caller's thread, and all invocations are blocking
	zstd_c_nb_workers = 400
	// Size of a compression job. This value is enforced only when nbWorkers >= 1.
	// Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
	// 0 means default, which is dynamically determined based on compression parameters.
	// Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
	// The minimum size is automatically and transparently enforced.
	zstd_c_job_size = 401
	// Control the overlap size, as a fraction of window size.
	// The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
	// It helps preserve compression ratio, while each job is compressed in parallel.
	// This value is enforced only when nbWorkers >= 1.
	// Larger values increase compression ratio, but decrease speed.
	// Possible values range from 0 to 9 :
	// - 0 means "default" : value will be determined by the library, depending on strategy
	// - 1 means "no overlap"
	// - 9 means "full overlap", using a full window size.
	// Each intermediate rank increases/decreases load size by a factor 2 :
	// 9: full window;  8: w/2;  7: w/4;  6: w/8;  5:w/16;  4: w/32;  3:w/64;  2:w/128;  1:no overlap;  0:default
	// default value varies between 6 and 9, depending on strategy
	zstd_c_overlap_log = 402
	// note : additional experimental parameters are also available
	// within the experimental section of the API.
	// At the time of this writing, they include :
	// zstd_c_rsyncable
	// zstd_c_format
	// zstd_c_force_max_window
	// zstd_c_force_attach_dict
	// zstd_c_literal_compression_mode
	// zstd_c_target_c_block_size
	// zstd_c_src_size_hint
	// zstd_c_enable_dedicated_dict_search
	// zstd_c_stable_in_buffer
	// zstd_c_stable_out_buffer
	// zstd_c_block_delimiters
	// zstd_c_validate_sequences
	// zstd_c_use_block_splitter
	// zstd_c_use_row_match_finder
	// zstd_c_prefetch_c_dict_tables
	// zstd_c_enable_seq_producer_fallback
	// zstd_c_max_block_size
	// Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
	// note : never ever use experimentalParam? names directly;
	//        also, the enums values themselves are unstable and can still change.
	//
	zstd_c_experimental_param1  = 500
	zstd_c_experimental_param2  = 10
	zstd_c_experimental_param3  = 1000
	zstd_c_experimental_param4  = 1001
	zstd_c_experimental_param5  = 1002
	zstd_c_experimental_param6  = 1003
	zstd_c_experimental_param7  = 1004
	zstd_c_experimental_param8  = 1005
	zstd_c_experimental_param9  = 1006
	zstd_c_experimental_param10 = 1007
	zstd_c_experimental_param11 = 1008
	zstd_c_experimental_param12 = 1009
	zstd_c_experimental_param13 = 1010
	zstd_c_experimental_param14 = 1011
	zstd_c_experimental_param15 = 1012
	zstd_c_experimental_param16 = 1013
	zstd_c_experimental_param17 = 1014
	zstd_c_experimental_param18 = 1015
	zstd_c_experimental_param19 = 1016
}

enum ZSTD_dParameter {
	// Select a size limit (in power of 2) beyond which
	// the streaming API will refuse to allocate memory buffer
	// in order to protect the host from unreasonable memory requirements.
	// This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode.
	// By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT).
	// Special: value 0 means "use default maximum windowLog".
	zstd_d_window_log_max = 100
	// note : additional experimental parameters are also available
	// within the experimental section of the API.
	// At the time of this writing, they include :
	// ZSTD_d_format
	// zstd_d_stable_out_buffer
	// zstd_d_force_ignore_checksum
	// zstd_d_ref_multipled_dicts
	// zstd_d_disable_huffman_assembly
	// Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
	// note : never ever use experimentalParam? names directly
	zstd_d_experimental_param1 = 1000
	zstd_d_experimental_param2 = 1001
	zstd_d_experimental_param3 = 1002
	zstd_d_experimental_param4 = 1003
	zstd_d_experimental_param5 = 1004
}

enum ZSTD_strategy {
	zstd_default  = 0
	zstd_fast     = 1
	zstd_dfast    = 2
	zstd_greedy   = 3
	zstd_lazy     = 4
	zstd_lazy2    = 5
	zstd_btlazy2  = 6
	zstd_btopt    = 7
	zstd_btultra  = 8
	zstd_btultra2 = 9
	// note : new strategies _might_ be added in the future. Only the order (from fast to strong) is guaranteed
}

struct CompressParams {
pub:
	compression_level int // 1~22
	nb_threads        int           = 1 // how many threads will be spawned to compress in parallel
	checksum_flag     bool          = true
	strategy          ZSTD_strategy = ZSTD_strategy.zstd_default
}

struct DecompressParams {
pub:
	window_log_max int
}

struct ZSTD_bounds {
	error       usize
	lower_bound int
	upper_bound int
}

struct ZSTD_inBuffer {
pub mut:
	src  voidptr
	size usize
	pos  usize
}

streaming compression

struct ZSTD_outBuffer {
pub mut:
	dst  voidptr
	size usize
	pos  usize
}