fix: README -> README.md

This commit is contained in:
tmtt 2022-08-09 23:28:25 +02:00
parent 43e68af625
commit 99b0a6292c
756 changed files with 323753 additions and 71 deletions

View file

@ -0,0 +1,78 @@
# ##########################################################################
# LZ4 oss fuzzer - Makefile
#
# GPL v2 License
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# You can contact the author at :
# - LZ4 homepage : http://www.lz4.org
# - LZ4 source repository : https://github.com/lz4/lz4
# ##########################################################################
# compress_fuzzer : OSS Fuzz test tool
# decompress_fuzzer : OSS Fuzz test tool
# ##########################################################################
LZ4DIR := ../lib
LIB_FUZZING_ENGINE ?=
DEBUGLEVEL?= 1
DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
LZ4_CFLAGS = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS)
LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \
-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
FUZZERS := \
compress_fuzzer \
decompress_fuzzer \
round_trip_fuzzer \
round_trip_stream_fuzzer \
compress_hc_fuzzer \
round_trip_hc_fuzzer \
compress_frame_fuzzer \
round_trip_frame_fuzzer \
decompress_frame_fuzzer
.PHONY: all
all: $(FUZZERS)
# Include a rule to build the static library if calling this target
# directly.
$(LZ4DIR)/liblz4.a:
$(MAKE) -C $(LZ4DIR) CFLAGS="$(LZ4_CFLAGS)" liblz4.a
%.o: %.c
$(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@
# Generic rule for generating fuzzers
ifeq ($(LIB_FUZZING_ENGINE),)
LIB_FUZZING_DEPS := standaloneengine.o
else
LIB_FUZZING_DEPS :=
endif
%_fuzzer: %_fuzzer.o lz4_helpers.o fuzz_data_producer.o $(LZ4DIR)/liblz4.a $(LIB_FUZZING_DEPS)
$(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT)
%_fuzzer_clean:
$(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o
.PHONY: clean
clean: compress_fuzzer_clean decompress_fuzzer_clean \
compress_frame_fuzzer_clean compress_hc_fuzzer_clean \
decompress_frame_fuzzer_clean round_trip_frame_fuzzer_clean \
round_trip_fuzzer_clean round_trip_hc_fuzzer_clean round_trip_stream_fuzzer_clean
$(MAKE) -C $(LZ4DIR) clean

View file

@ -0,0 +1,48 @@
/**
* This fuzz target attempts to compress the fuzzed data with the simple
* compression function with an output buffer that may be too small to
* ensure that the compressor never crashes.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "lz4.h"
#include "lz4frame.h"
#include "lz4_helpers.h"
#include "fuzz_data_producer.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const compressBound = LZ4F_compressFrameBound(size, &prefs);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst!=NULL);
FUZZ_ASSERT(rt!=NULL);
/* If compression succeeds it must round trip correctly. */
size_t const dstSize =
LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs);
if (!LZ4F_isError(dstSize)) {
size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
}
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,58 @@
/**
* This fuzz target attempts to compress the fuzzed data with the simple
* compression function with an output buffer that may be too small to
* ensure that the compressor never crashes.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const compressBound = LZ4_compressBound(size);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* If compression succeeds it must round trip correctly. */
{
int const dstSize = LZ4_compress_default((const char*)data, dst,
size, dstCapacity);
if (dstSize > 0) {
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
}
}
if (dstCapacity > 0) {
/* Compression succeeds and must round trip correctly. */
int compressedSize = size;
int const dstSize = LZ4_compress_destSize((const char*)data, dst,
&compressedSize, dstCapacity);
FUZZ_ASSERT(dstSize > 0);
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
}
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,64 @@
/**
* This fuzz target attempts to compress the fuzzed data with the simple
* compression function with an output buffer that may be too small to
* ensure that the compressor never crashes.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
#include "lz4hc.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size_t const levelSeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, size);
int const level = FUZZ_getRange_from_uint32(levelSeed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* If compression succeeds it must round trip correctly. */
{
int const dstSize = LZ4_compress_HC((const char*)data, dst, size,
dstCapacity, level);
if (dstSize > 0) {
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
}
}
if (dstCapacity > 0) {
/* Compression succeeds and must round trip correctly. */
void* state = malloc(LZ4_sizeofStateHC());
FUZZ_ASSERT(state);
int compressedSize = size;
int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data,
dst, &compressedSize,
dstCapacity, level);
FUZZ_ASSERT(dstSize > 0);
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!");
free(state);
}
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,75 @@
/**
* This fuzz target attempts to decompress the fuzzed data with the simple
* decompression function to ensure the decompressor never crashes.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
#define LZ4F_STATIC_LINKING_ONLY
#include "lz4frame.h"
#include "lz4_helpers.h"
static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const void* dict, size_t dictSize,
const LZ4F_decompressOptions_t* opts)
{
LZ4F_resetDecompressionContext(dctx);
if (dictSize == 0)
LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts);
else
LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize,
dict, dictSize, opts);
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size_t const dictSizeSeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = FUZZ_getRange_from_uint32(
dstCapacitySeed, 0, 4 * size);
size_t const largeDictSize = 64 * 1024;
size_t const dictSize = FUZZ_getRange_from_uint32(
dictSizeSeed, 0, largeDictSize);
char* const dst = (char*)malloc(dstCapacity);
char* const dict = (char*)malloc(dictSize);
LZ4F_decompressOptions_t opts;
LZ4F_dctx* dctx;
LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
FUZZ_ASSERT(dctx);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(dict);
/* Prepare the dictionary. The data doesn't matter for decompression. */
memset(dict, 0, dictSize);
/* Decompress using multiple configurations. */
memset(&opts, 0, sizeof(opts));
opts.stableDst = 0;
decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
opts.stableDst = 1;
decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts);
opts.stableDst = 0;
decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
opts.stableDst = 1;
decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts);
LZ4F_freeDecompressionContext(dctx);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,62 @@
/**
* This fuzz target attempts to decompress the fuzzed data with the simple
* decompression function to ensure the decompressor never crashes.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, 4 * size);
size_t const smallDictSize = size + 1;
size_t const largeDictSize = 64 * 1024 - 1;
size_t const dictSize = MAX(smallDictSize, largeDictSize);
char* const dst = (char*)malloc(dstCapacity);
char* const dict = (char*)malloc(dictSize + size);
char* const largeDict = dict;
char* const dataAfterDict = dict + dictSize;
char* const smallDict = dataAfterDict - smallDictSize;
FUZZ_ASSERT(dst);
FUZZ_ASSERT(dict);
/* Prepare the dictionary. The data doesn't matter for decompression. */
memset(dict, 0, dictSize);
memcpy(dataAfterDict, data, size);
/* Decompress using each possible dictionary configuration. */
/* No dictionary. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, NULL, 0);
/* Small external dictonary. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, smallDict, smallDictSize);
/* Large external dictionary. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, largeDict, largeDictSize);
/* Small prefix. */
LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size,
dstCapacity, smallDict, smallDictSize);
/* Large prefix. */
LZ4_decompress_safe_usingDict((char const*)data, dst, size,
dstCapacity, largeDict, largeDictSize);
/* Partial decompression. */
LZ4_decompress_safe_partial((char const*)data, dst, size,
dstCapacity, dstCapacity);
free(dst);
free(dict);
FUZZ_dataProducer_free(producer);
return 0;
}

48
ta6ob/lz4/ossfuzz/fuzz.h Normal file
View file

@ -0,0 +1,48 @@
/**
* Fuzz target interface.
* Fuzz targets have some common parameters passed as macros during compilation.
* Check the documentation for each individual fuzzer for more parameters.
*
* @param FUZZ_RNG_SEED_SIZE:
* The number of bytes of the source to look at when constructing a seed
* for the deterministic RNG. These bytes are discarded before passing
* the data to lz4 functions. Every fuzzer initializes the RNG exactly
* once before doing anything else, even if it is unused.
* Default: 4.
* @param LZ4_DEBUG:
* This is a parameter for the lz4 library. Defining `LZ4_DEBUG=1`
* enables assert() statements in the lz4 library. Higher levels enable
* logging, so aren't recommended. Defining `LZ4_DEBUG=1` is
* recommended.
* @param LZ4_FORCE_MEMORY_ACCESS:
* This flag controls how the zstd library accesses unaligned memory.
* It can be undefined, or 0 through 2. If it is undefined, it selects
* the method to use based on the compiler. If testing with UBSAN set
* MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
* @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
* This is the canonical flag to enable deterministic builds for fuzzing.
* Changes to zstd for fuzzing are gated behind this define.
* It is recommended to define this when building zstd for fuzzing.
*/
#ifndef FUZZ_H
#define FUZZ_H
#ifndef FUZZ_RNG_SEED_SIZE
# define FUZZ_RNG_SEED_SIZE 4
#endif
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size);
#ifdef __cplusplus
}
#endif
#endif

View file

@ -0,0 +1,77 @@
#include "fuzz_data_producer.h"
struct FUZZ_dataProducer_s{
const uint8_t *data;
size_t size;
};
FUZZ_dataProducer_t* FUZZ_dataProducer_create(const uint8_t* data, size_t size) {
FUZZ_dataProducer_t* const producer = malloc(sizeof(FUZZ_dataProducer_t));
FUZZ_ASSERT(producer != NULL);
producer->data = data;
producer->size = size;
return producer;
}
void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); }
uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer) {
const uint8_t* data = producer->data;
const size_t size = producer->size;
if (size == 0) {
return 0;
} else if (size < 4) {
producer->size -= 1;
return (uint32_t)data[size - 1];
} else {
producer->size -= 4;
return *(data + size - 4);
}
}
uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max)
{
uint32_t range = max - min;
if (range == 0xffffffff) {
return seed;
}
return min + seed % (range + 1);
}
uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t* producer,
uint32_t min, uint32_t max)
{
size_t const seed = FUZZ_dataProducer_retrieve32(producer);
return FUZZ_getRange_from_uint32(seed, min, max);
}
LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer)
{
LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO;
info.blockSizeID = FUZZ_dataProducer_range32(producer, LZ4F_max64KB - 1, LZ4F_max4MB);
if (info.blockSizeID < LZ4F_max64KB) {
info.blockSizeID = LZ4F_default;
}
info.blockMode = FUZZ_dataProducer_range32(producer, LZ4F_blockLinked, LZ4F_blockIndependent);
info.contentChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noContentChecksum,
LZ4F_contentChecksumEnabled);
info.blockChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noBlockChecksum,
LZ4F_blockChecksumEnabled);
return info;
}
LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer)
{
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
prefs.frameInfo = FUZZ_dataProducer_frameInfo(producer);
prefs.compressionLevel = FUZZ_dataProducer_range32(producer, 0, LZ4HC_CLEVEL_MAX + 3) - 3;
prefs.autoFlush = FUZZ_dataProducer_range32(producer, 0, 1);
prefs.favorDecSpeed = FUZZ_dataProducer_range32(producer, 0, 1);
return prefs;
}
size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){
return producer->size;
}

View file

@ -0,0 +1,36 @@
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "fuzz_helpers.h"
#include "lz4frame.h"
#include "lz4hc.h"
/* Struct used for maintaining the state of the data */
typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t;
/* Returns a data producer state struct. Use for producer initialization. */
FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size);
/* Frees the data producer */
void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer);
/* Returns 32 bits from the end of data */
uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer);
/* Returns value between [min, max] */
uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max);
/* Combination of above two functions for non adaptive use cases. ie where size is not involved */
uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t *producer, uint32_t min,
uint32_t max);
/* Returns lz4 preferences */
LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer);
/* Returns lz4 frame info */
LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer);
/* Returns the size of the remaining bytes of data in the producer */
size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer);

View file

@ -0,0 +1,94 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
* Helper functions for fuzzing.
*/
#ifndef FUZZ_HELPERS_H
#define FUZZ_HELPERS_H
#include "fuzz.h"
#include "xxhash.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
#define LZ4_COMMONDEFS_ONLY
#ifndef LZ4_SRC_INCLUDED
#include "lz4.c" /* LZ4_count, constants, mem */
#endif
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
#define FUZZ_QUOTE_IMPL(str) #str
#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str)
/**
* Asserts for fuzzing that are always enabled.
*/
#define FUZZ_ASSERT_MSG(cond, msg) \
((cond) ? (void)0 \
: (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \
__LINE__, FUZZ_QUOTE(cond), (msg)), \
abort()))
#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), "");
#if defined(__GNUC__)
#define FUZZ_STATIC static __inline __attribute__((unused))
#elif defined(__cplusplus) || \
(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
#define FUZZ_STATIC static inline
#elif defined(_MSC_VER)
#define FUZZ_STATIC static __inline
#else
#define FUZZ_STATIC static
#endif
/**
* Deterministically constructs a seed based on the fuzz input.
* Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input.
*/
FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) {
uint8_t const *data = *src;
size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size);
*size -= toHash;
*src += toHash;
return XXH32(data, toHash, 0);
}
#define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r))))
FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) {
static const uint32_t prime1 = 2654435761U;
static const uint32_t prime2 = 2246822519U;
uint32_t rand32 = *state;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZZ_rotl32(rand32, 13);
*state = rand32;
return rand32 >> 5;
}
/* Returns a random numer in the range [min, max]. */
FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) {
uint32_t random = FUZZ_rand(state);
return min + (random % (max - min + 1));
}
#ifdef __cplusplus
}
#endif
#endif

View file

@ -0,0 +1,51 @@
#include "fuzz_helpers.h"
#include "lz4_helpers.h"
#include "lz4hc.h"
LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed)
{
LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO;
info.blockSizeID = FUZZ_rand32(seed, LZ4F_max64KB - 1, LZ4F_max4MB);
if (info.blockSizeID < LZ4F_max64KB) {
info.blockSizeID = LZ4F_default;
}
info.blockMode = FUZZ_rand32(seed, LZ4F_blockLinked, LZ4F_blockIndependent);
info.contentChecksumFlag = FUZZ_rand32(seed, LZ4F_noContentChecksum,
LZ4F_contentChecksumEnabled);
info.blockChecksumFlag = FUZZ_rand32(seed, LZ4F_noBlockChecksum,
LZ4F_blockChecksumEnabled);
return info;
}
LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed)
{
LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES;
prefs.frameInfo = FUZZ_randomFrameInfo(seed);
prefs.compressionLevel = FUZZ_rand32(seed, 0, LZ4HC_CLEVEL_MAX + 3) - 3;
prefs.autoFlush = FUZZ_rand32(seed, 0, 1);
prefs.favorDecSpeed = FUZZ_rand32(seed, 0, 1);
return prefs;
}
size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity,
const void* src, const size_t srcSize)
{
LZ4F_decompressOptions_t opts;
memset(&opts, 0, sizeof(opts));
opts.stableDst = 1;
LZ4F_dctx* dctx;
LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
FUZZ_ASSERT(dctx);
size_t dstSize = dstCapacity;
size_t srcConsumed = srcSize;
size_t const rc =
LZ4F_decompress(dctx, dst, &dstSize, src, &srcConsumed, &opts);
FUZZ_ASSERT(!LZ4F_isError(rc));
FUZZ_ASSERT(rc == 0);
FUZZ_ASSERT(srcConsumed == srcSize);
LZ4F_freeDecompressionContext(dctx);
return dstSize;
}

View file

@ -0,0 +1,13 @@
#ifndef LZ4_HELPERS
#define LZ4_HELPERS
#include "lz4frame.h"
LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed);
LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed);
size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity,
const void* src, const size_t srcSize);
#endif /* LZ4_HELPERS */

23
ta6ob/lz4/ossfuzz/ossfuzz.sh Executable file
View file

@ -0,0 +1,23 @@
#!/bin/bash -eu
# This script is called by the oss-fuzz main project when compiling the fuzz
# targets. This script is regression tested by travisoss.sh.
# Save off the current folder as the build root.
export BUILD_ROOT=$PWD
echo "CC: $CC"
echo "CXX: $CXX"
echo "LIB_FUZZING_ENGINE: $LIB_FUZZING_ENGINE"
echo "CFLAGS: $CFLAGS"
echo "CXXFLAGS: $CXXFLAGS"
echo "OUT: $OUT"
export MAKEFLAGS+="-j$(nproc)"
pushd ossfuzz
make V=1 all
popd
# Copy the fuzzers to the target directory.
cp -v ossfuzz/*_fuzzer $OUT/

View file

@ -0,0 +1,43 @@
/**
* This fuzz target performs a lz4 round-trip test (compress & decompress),
* compares the result with the original, and calls abort() on corruption.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "lz4.h"
#include "lz4frame.h"
#include "lz4_helpers.h"
#include "fuzz_data_producer.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t* producer = FUZZ_dataProducer_create(data, size);
LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(FUZZ_dataProducer_remainingBytes(producer));
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* Compression must succeed and round trip correctly. */
size_t const dstSize =
LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs);
FUZZ_ASSERT(!LZ4F_isError(dstSize));
size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,57 @@
/**
* This fuzz target performs a lz4 round-trip test (compress & decompress),
* compares the result with the original, and calls abort() on corruption.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "lz4.h"
#include "fuzz_data_producer.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
size_t const partialCapacitySeed = FUZZ_dataProducer_retrieve32(producer);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size);
size_t const dstCapacity = LZ4_compressBound(size);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* Compression must succeed and round trip correctly. */
int const dstSize = LZ4_compress_default((const char*)data, dst,
size, dstCapacity);
FUZZ_ASSERT(dstSize > 0);
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
/* Partial decompression must succeed. */
{
char* const partial = (char*)malloc(partialCapacity);
FUZZ_ASSERT(partial);
int const partialSize = LZ4_decompress_safe_partial(
dst, partial, dstSize, partialCapacity, partialCapacity);
FUZZ_ASSERT(partialSize >= 0);
FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size");
FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!");
free(partial);
}
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,44 @@
/**
* This fuzz target performs a lz4 round-trip test (compress & decompress),
* compares the result with the original, and calls abort() on corruption.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#include "fuzz_data_producer.h"
#include "lz4.h"
#include "lz4hc.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size);
int const level = FUZZ_dataProducer_range32(producer,
LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
size = FUZZ_dataProducer_remainingBytes(producer);
size_t const dstCapacity = LZ4_compressBound(size);
char* const dst = (char*)malloc(dstCapacity);
char* const rt = (char*)malloc(size);
FUZZ_ASSERT(dst);
FUZZ_ASSERT(rt);
/* Compression must succeed and round trip correctly. */
int const dstSize = LZ4_compress_HC((const char*)data, dst, size,
dstCapacity, level);
FUZZ_ASSERT(dstSize > 0);
int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size);
FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size");
FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!");
free(dst);
free(rt);
FUZZ_dataProducer_free(producer);
return 0;
}

View file

@ -0,0 +1,302 @@
/**
* This fuzz target performs a lz4 streaming round-trip test
* (compress & decompress), compares the result with the original, and calls
* abort() on corruption.
*/
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "fuzz_helpers.h"
#define LZ4_STATIC_LINKING_ONLY
#include "lz4.h"
#define LZ4_HC_STATIC_LINKING_ONLY
#include "lz4hc.h"
typedef struct {
char const* buf;
size_t size;
size_t pos;
} const_cursor_t;
typedef struct {
char* buf;
size_t size;
size_t pos;
} cursor_t;
typedef struct {
LZ4_stream_t* cstream;
LZ4_streamHC_t* cstreamHC;
LZ4_streamDecode_t* dstream;
const_cursor_t data;
cursor_t compressed;
cursor_t roundTrip;
uint32_t seed;
int level;
} state_t;
cursor_t cursor_create(size_t size)
{
cursor_t cursor;
cursor.buf = (char*)malloc(size);
cursor.size = size;
cursor.pos = 0;
FUZZ_ASSERT(cursor.buf);
return cursor;
}
typedef void (*round_trip_t)(state_t* state);
void cursor_free(cursor_t cursor)
{
free(cursor.buf);
}
state_t state_create(char const* data, size_t size, uint32_t seed)
{
state_t state;
state.seed = seed;
state.data.buf = (char const*)data;
state.data.size = size;
state.data.pos = 0;
/* Extra margin because we are streaming. */
state.compressed = cursor_create(1024 + 2 * LZ4_compressBound(size));
state.roundTrip = cursor_create(size);
state.cstream = LZ4_createStream();
FUZZ_ASSERT(state.cstream);
state.cstreamHC = LZ4_createStreamHC();
FUZZ_ASSERT(state.cstream);
state.dstream = LZ4_createStreamDecode();
FUZZ_ASSERT(state.dstream);
return state;
}
void state_free(state_t state)
{
cursor_free(state.compressed);
cursor_free(state.roundTrip);
LZ4_freeStream(state.cstream);
LZ4_freeStreamHC(state.cstreamHC);
LZ4_freeStreamDecode(state.dstream);
}
static void state_reset(state_t* state, uint32_t seed)
{
state->level = FUZZ_rand32(&seed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX);
LZ4_resetStream_fast(state->cstream);
LZ4_resetStreamHC_fast(state->cstreamHC, state->level);
LZ4_setStreamDecode(state->dstream, NULL, 0);
state->data.pos = 0;
state->compressed.pos = 0;
state->roundTrip.pos = 0;
state->seed = seed;
}
static void state_decompress(state_t* state, char const* src, int srcSize)
{
char* dst = state->roundTrip.buf + state->roundTrip.pos;
int const dstCapacity = state->roundTrip.size - state->roundTrip.pos;
int const dSize = LZ4_decompress_safe_continue(state->dstream, src, dst,
srcSize, dstCapacity);
FUZZ_ASSERT(dSize >= 0);
state->roundTrip.pos += dSize;
}
static void state_checkRoundTrip(state_t const* state)
{
char const* data = state->data.buf;
size_t const size = state->data.size;
FUZZ_ASSERT_MSG(size == state->roundTrip.pos, "Incorrect size!");
FUZZ_ASSERT_MSG(!memcmp(data, state->roundTrip.buf, size), "Corruption!");
}
/**
* Picks a dictionary size and trims the dictionary off of the data.
* We copy the dictionary to the roundTrip so our validation passes.
*/
static size_t state_trimDict(state_t* state)
{
/* 64 KB is the max dict size, allow slightly beyond that to test trim. */
uint32_t maxDictSize = MIN(70 * 1024, state->data.size);
size_t const dictSize = FUZZ_rand32(&state->seed, 0, maxDictSize);
DEBUGLOG(2, "dictSize = %zu", dictSize);
FUZZ_ASSERT(state->data.pos == 0);
FUZZ_ASSERT(state->roundTrip.pos == 0);
memcpy(state->roundTrip.buf, state->data.buf, dictSize);
state->data.pos += dictSize;
state->roundTrip.pos += dictSize;
return dictSize;
}
static void state_prefixRoundTrip(state_t* state)
{
while (state->data.pos != state->data.size) {
char const* src = state->data.buf + state->data.pos;
char* dst = state->compressed.buf + state->compressed.pos;
int const srcRemaining = state->data.size - state->data.pos;
int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
int const dstCapacity = state->compressed.size - state->compressed.pos;
int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst,
srcSize, dstCapacity, 0);
FUZZ_ASSERT(cSize > 0);
state->data.pos += srcSize;
state->compressed.pos += cSize;
state_decompress(state, dst, cSize);
}
}
static void state_extDictRoundTrip(state_t* state)
{
int i = 0;
cursor_t data2 = cursor_create(state->data.size);
memcpy(data2.buf, state->data.buf, state->data.size);
while (state->data.pos != state->data.size) {
char const* data = (i++ & 1) ? state->data.buf : data2.buf;
char const* src = data + state->data.pos;
char* dst = state->compressed.buf + state->compressed.pos;
int const srcRemaining = state->data.size - state->data.pos;
int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
int const dstCapacity = state->compressed.size - state->compressed.pos;
int const cSize = LZ4_compress_fast_continue(state->cstream, src, dst,
srcSize, dstCapacity, 0);
FUZZ_ASSERT(cSize > 0);
state->data.pos += srcSize;
state->compressed.pos += cSize;
state_decompress(state, dst, cSize);
}
cursor_free(data2);
}
static void state_randomRoundTrip(state_t* state, round_trip_t rt0,
round_trip_t rt1)
{
if (FUZZ_rand32(&state->seed, 0, 1)) {
rt0(state);
} else {
rt1(state);
}
}
static void state_loadDictRoundTrip(state_t* state)
{
char const* dict = state->data.buf;
size_t const dictSize = state_trimDict(state);
LZ4_loadDict(state->cstream, dict, dictSize);
LZ4_setStreamDecode(state->dstream, dict, dictSize);
state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip);
}
static void state_attachDictRoundTrip(state_t* state)
{
char const* dict = state->data.buf;
size_t const dictSize = state_trimDict(state);
LZ4_stream_t* dictStream = LZ4_createStream();
LZ4_loadDict(dictStream, dict, dictSize);
LZ4_attach_dictionary(state->cstream, dictStream);
LZ4_setStreamDecode(state->dstream, dict, dictSize);
state_randomRoundTrip(state, state_prefixRoundTrip, state_extDictRoundTrip);
LZ4_freeStream(dictStream);
}
static void state_prefixHCRoundTrip(state_t* state)
{
while (state->data.pos != state->data.size) {
char const* src = state->data.buf + state->data.pos;
char* dst = state->compressed.buf + state->compressed.pos;
int const srcRemaining = state->data.size - state->data.pos;
int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
int const dstCapacity = state->compressed.size - state->compressed.pos;
int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst,
srcSize, dstCapacity);
FUZZ_ASSERT(cSize > 0);
state->data.pos += srcSize;
state->compressed.pos += cSize;
state_decompress(state, dst, cSize);
}
}
static void state_extDictHCRoundTrip(state_t* state)
{
int i = 0;
cursor_t data2 = cursor_create(state->data.size);
DEBUGLOG(2, "extDictHC");
memcpy(data2.buf, state->data.buf, state->data.size);
while (state->data.pos != state->data.size) {
char const* data = (i++ & 1) ? state->data.buf : data2.buf;
char const* src = data + state->data.pos;
char* dst = state->compressed.buf + state->compressed.pos;
int const srcRemaining = state->data.size - state->data.pos;
int const srcSize = FUZZ_rand32(&state->seed, 0, srcRemaining);
int const dstCapacity = state->compressed.size - state->compressed.pos;
int const cSize = LZ4_compress_HC_continue(state->cstreamHC, src, dst,
srcSize, dstCapacity);
FUZZ_ASSERT(cSize > 0);
DEBUGLOG(2, "srcSize = %d", srcSize);
state->data.pos += srcSize;
state->compressed.pos += cSize;
state_decompress(state, dst, cSize);
}
cursor_free(data2);
}
static void state_loadDictHCRoundTrip(state_t* state)
{
char const* dict = state->data.buf;
size_t const dictSize = state_trimDict(state);
LZ4_loadDictHC(state->cstreamHC, dict, dictSize);
LZ4_setStreamDecode(state->dstream, dict, dictSize);
state_randomRoundTrip(state, state_prefixHCRoundTrip,
state_extDictHCRoundTrip);
}
static void state_attachDictHCRoundTrip(state_t* state)
{
char const* dict = state->data.buf;
size_t const dictSize = state_trimDict(state);
LZ4_streamHC_t* dictStream = LZ4_createStreamHC();
LZ4_setCompressionLevel(dictStream, state->level);
LZ4_loadDictHC(dictStream, dict, dictSize);
LZ4_attach_HC_dictionary(state->cstreamHC, dictStream);
LZ4_setStreamDecode(state->dstream, dict, dictSize);
state_randomRoundTrip(state, state_prefixHCRoundTrip,
state_extDictHCRoundTrip);
LZ4_freeStreamHC(dictStream);
}
round_trip_t roundTrips[] = {
&state_prefixRoundTrip,
&state_extDictRoundTrip,
&state_loadDictRoundTrip,
&state_attachDictRoundTrip,
&state_prefixHCRoundTrip,
&state_extDictHCRoundTrip,
&state_loadDictHCRoundTrip,
&state_attachDictHCRoundTrip,
};
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
uint32_t seed = FUZZ_seed(&data, &size);
state_t state = state_create((char const*)data, size, seed);
const int n = sizeof(roundTrips) / sizeof(round_trip_t);
int i;
for (i = 0; i < n; ++i) {
DEBUGLOG(2, "Round trip %d", i);
state_reset(&state, seed);
roundTrips[i](&state);
state_checkRoundTrip(&state);
}
state_free(state);
return 0;
}

View file

@ -0,0 +1,74 @@
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "fuzz.h"
/**
* Main procedure for standalone fuzzing engine.
*
* Reads filenames from the argument array. For each filename, read the file
* into memory and then call the fuzzing interface with the data.
*/
int main(int argc, char **argv)
{
int ii;
for(ii = 1; ii < argc; ii++)
{
FILE *infile;
printf("[%s] ", argv[ii]);
/* Try and open the file. */
infile = fopen(argv[ii], "rb");
if(infile)
{
uint8_t *buffer = NULL;
size_t buffer_len;
printf("Opened.. ");
/* Get the length of the file. */
fseek(infile, 0L, SEEK_END);
buffer_len = ftell(infile);
/* Reset the file indicator to the beginning of the file. */
fseek(infile, 0L, SEEK_SET);
/* Allocate a buffer for the file contents. */
buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t));
if(buffer)
{
/* Read all the text from the file into the buffer. */
fread(buffer, sizeof(uint8_t), buffer_len, infile);
printf("Read %zu bytes, fuzzing.. ", buffer_len);
/* Call the fuzzer with the data. */
LLVMFuzzerTestOneInput(buffer, buffer_len);
printf("complete !!");
/* Free the buffer as it's no longer needed. */
free(buffer);
buffer = NULL;
}
else
{
fprintf(stderr,
"[%s] Failed to allocate %zu bytes \n",
argv[ii],
buffer_len);
}
/* Close the file as it's no longer needed. */
fclose(infile);
infile = NULL;
}
else
{
/* Failed to open the file. Maybe wrong name or wrong permissions? */
fprintf(stderr, "[%s] Open failed. \n", argv[ii]);
}
printf("\n");
}
}

26
ta6ob/lz4/ossfuzz/travisoss.sh Executable file
View file

@ -0,0 +1,26 @@
#!/bin/bash
set -ex
# Clone the oss-fuzz repository
git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz
if [[ ! -d /tmp/ossfuzz/projects/lz4 ]]
then
echo "Could not find the lz4 project in ossfuzz"
exit 1
fi
# Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis.
if [ "x${TRAVIS_PULL_REQUEST}" = "xfalse" ]
then
sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_BRANCH} https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
else
sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_PULL_REQUEST_BRANCH} https://github.com/${TRAVIS_PULL_REQUEST_SLUG}.git@" /tmp/ossfuzz/projects/lz4/Dockerfile
fi
# Try and build the fuzzers
pushd /tmp/ossfuzz
python infra/helper.py build_image --pull lz4
python infra/helper.py build_fuzzers lz4
popd