feat: 9.5.9
This commit is contained in:
tmtt
parent
cb1753732b
commit
35f43a7909
1084 changed files with 558985 additions and 0 deletions
22
lz4/tests/.gitignore
vendored
Normal file
22
lz4/tests/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
|
||||
# build artefacts
|
||||
datagen
|
||||
frametest
|
||||
frametest32
|
||||
fullbench
|
||||
fullbench32
|
||||
fuzzer
|
||||
fuzzer32
|
||||
fasttest
|
||||
roundTripTest
|
||||
checkTag
|
||||
checkFrame
|
||||
decompress-partial
|
||||
|
||||
# test artefacts
|
||||
tmp*
|
||||
versionsTest
|
||||
lz4_all.c
|
||||
|
||||
# local tests
|
||||
afl
|
||||
339
lz4/tests/COPYING
Normal file
339
lz4/tests/COPYING
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
||||
544
lz4/tests/Makefile
Normal file
544
lz4/tests/Makefile
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
# ##########################################################################
|
||||
# LZ4 programs - Makefile
|
||||
# Copyright (C) Yann Collet 2011-present
|
||||
#
|
||||
# GPL v2 License
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# You can contact the author at :
|
||||
# - LZ4 homepage : http://www.lz4.org
|
||||
# - LZ4 source repository : https://github.com/lz4/lz4
|
||||
# ##########################################################################
|
||||
# fuzzer : Test tool, to check lz4 integrity on target platform
|
||||
# frametest : Test tool, to check lz4frame integrity on target platform
|
||||
# fullbench : Precisely measure speed for each LZ4 function variant
|
||||
# datagen : generates synthetic data samples for tests & benchmarks
|
||||
# ##########################################################################
|
||||
|
||||
LZ4DIR := ../lib
|
||||
PRGDIR := ../programs
|
||||
TESTDIR := versionsTest
|
||||
PYTHON ?= python3
|
||||
|
||||
DEBUGLEVEL?= 1
|
||||
DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL)
|
||||
CFLAGS ?= -O3 # can select custom optimization flags. Example : CFLAGS=-O2 make
|
||||
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow \
|
||||
-Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
|
||||
-Wpointer-arith -Wstrict-aliasing=1
|
||||
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
|
||||
CPPFLAGS+= -I$(LZ4DIR) -I$(PRGDIR) -DXXH_NAMESPACE=LZ4_
|
||||
FLAGS = $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
|
||||
|
||||
include ../Makefile.inc
|
||||
|
||||
LZ4 := $(PRGDIR)/lz4$(EXT)
|
||||
|
||||
|
||||
# Default test parameters
|
||||
TEST_FILES := COPYING
|
||||
FUZZER_TIME := -T90s
|
||||
NB_LOOPS ?= -i1
|
||||
|
||||
.PHONY: default
|
||||
default: all
|
||||
|
||||
all: fullbench fuzzer frametest roundTripTest datagen checkFrame decompress-partial
|
||||
|
||||
all32: CFLAGS+=-m32
|
||||
all32: all
|
||||
|
||||
lz4:
|
||||
$(MAKE) -C $(PRGDIR) $@ CFLAGS="$(CFLAGS)"
|
||||
|
||||
lib liblz4.pc:
|
||||
$(MAKE) -C $(LZ4DIR) $@ CFLAGS="$(CFLAGS)"
|
||||
|
||||
lz4c unlz4 lz4cat: lz4
|
||||
$(LN_SF) $(LZ4) $(PRGDIR)/$@
|
||||
|
||||
lz4c32: # create a 32-bits version for 32/64 interop tests
|
||||
$(MAKE) -C $(PRGDIR) $@ CFLAGS="-m32 $(CFLAGS)"
|
||||
|
||||
%.o : $(LZ4DIR)/%.c $(LZ4DIR)/%.h
|
||||
$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@
|
||||
|
||||
fullbench : DEBUGLEVEL=0
|
||||
fullbench : lz4.o lz4hc.o lz4frame.o xxhash.o fullbench.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
$(LZ4DIR)/liblz4.a:
|
||||
$(MAKE) -C $(LZ4DIR) liblz4.a
|
||||
|
||||
fullbench-lib: fullbench.c $(LZ4DIR)/liblz4.a
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
fullbench-dll: fullbench.c $(LZ4DIR)/xxhash.c
|
||||
$(MAKE) -C $(LZ4DIR) liblz4
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(LZ4DIR)/dll/$(LIBLZ4).dll
|
||||
|
||||
# test LZ4_USER_MEMORY_FUNCTIONS
|
||||
fullbench-wmalloc: CPPFLAGS += -DLZ4_USER_MEMORY_FUNCTIONS
|
||||
fullbench-wmalloc: fullbench
|
||||
|
||||
fuzzer : lz4.o lz4hc.o xxhash.o fuzzer.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
frametest: lz4frame.o lz4.o lz4hc.o xxhash.o frametest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
roundTripTest : lz4.o lz4hc.o xxhash.o roundTripTest.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
datagen : $(PRGDIR)/datagen.c datagencli.c
|
||||
$(CC) $(FLAGS) -I$(PRGDIR) $^ -o $@$(EXT)
|
||||
|
||||
checkFrame : lz4frame.o lz4.o lz4hc.o xxhash.o checkFrame.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
decompress-partial: lz4.o decompress-partial.c
|
||||
$(CC) $(FLAGS) $^ -o $@$(EXT)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
@$(MAKE) -C $(LZ4DIR) $@ > $(VOID)
|
||||
@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
|
||||
@$(RM) -rf core *.o *.test tmp* \
|
||||
fullbench-dll$(EXT) fullbench-lib$(EXT) \
|
||||
fullbench$(EXT) fullbench32$(EXT) \
|
||||
fuzzer$(EXT) fuzzer32$(EXT) \
|
||||
frametest$(EXT) frametest32$(EXT) \
|
||||
fasttest$(EXT) roundTripTest$(EXT) \
|
||||
datagen$(EXT) checkTag$(EXT) \
|
||||
frameTest$(EXT) decompress-partial$(EXT) \
|
||||
lz4_all.c
|
||||
@$(RM) -rf $(TESTDIR)
|
||||
@echo Cleaning completed
|
||||
|
||||
.PHONY: versionsTest
|
||||
versionsTest:
|
||||
$(PYTHON) test-lz4-versions.py
|
||||
|
||||
.PHONY: listTest
|
||||
listTest: lz4
|
||||
QEMU_SYS=$(QEMU_SYS) $(PYTHON) test-lz4-list.py
|
||||
|
||||
checkTag: checkTag.c $(LZ4DIR)/lz4.h
|
||||
$(CC) $(FLAGS) $< -o $@$(EXT)
|
||||
|
||||
#-----------------------------------------------------------------------------
|
||||
# validated only for Linux, OSX, BSD, Hurd and Solaris targets
|
||||
#-----------------------------------------------------------------------------
|
||||
ifeq ($(POSIX_ENV),Yes)
|
||||
|
||||
MD5:=md5sum
|
||||
ifneq (,$(filter $(shell uname), Darwin ))
|
||||
MD5:=md5 -r
|
||||
endif
|
||||
|
||||
# note : we should probably settle on a single compare utility
|
||||
CMP:=cmp
|
||||
DIFF:=diff
|
||||
ifneq (,$(filter $(shell uname),SunOS))
|
||||
DIFF:=gdiff
|
||||
endif
|
||||
|
||||
CAT:=cat
|
||||
DD:=dd
|
||||
DATAGEN:=./datagen
|
||||
|
||||
.PHONY: list
|
||||
list:
|
||||
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
|
||||
|
||||
.PHONY: check
|
||||
check: test-lz4-essentials
|
||||
|
||||
.PHONY: test
|
||||
test: test-lz4 test-lz4c test-frametest test-fullbench test-fuzzer test-install test-amalgamation listTest test-decompress-partial
|
||||
|
||||
.PHONY: test32
|
||||
test32: CFLAGS+=-m32
|
||||
test32: test
|
||||
|
||||
test-amalgamation: lz4_all.o
|
||||
|
||||
lz4_all.c: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c
|
||||
$(CAT) $^ > $@
|
||||
|
||||
test-install: lz4 lib liblz4.pc
|
||||
lz4_root=.. ./test_install.sh
|
||||
|
||||
test-lz4-sparse: lz4 datagen
|
||||
@echo "\n ---- test sparse file support ----"
|
||||
$(DATAGEN) -g5M -P100 > tmplsdg5M
|
||||
$(LZ4) -B4D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB4
|
||||
$(DIFF) -s tmplsdg5M tmplscB4
|
||||
$(LZ4) -B5D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB5
|
||||
$(DIFF) -s tmplsdg5M tmplscB5
|
||||
$(LZ4) -B6D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB6
|
||||
$(DIFF) -s tmplsdg5M tmplscB6
|
||||
$(LZ4) -B7D tmplsdg5M -c | $(LZ4) -dv --sparse > tmplscB7
|
||||
$(DIFF) -s tmplsdg5M tmplscB7
|
||||
$(LZ4) tmplsdg5M -c | $(LZ4) -dv --no-sparse > tmplsnosparse
|
||||
$(DIFF) -s tmplsdg5M tmplsnosparse
|
||||
ls -ls tmpls*
|
||||
$(DATAGEN) -s1 -g1200007 -P100 | $(LZ4) | $(LZ4) -dv --sparse > tmplsodd # Odd size file (to generate non-full last block)
|
||||
$(DATAGEN) -s1 -g1200007 -P100 | $(DIFF) -s - tmplsodd
|
||||
ls -ls tmplsodd
|
||||
@$(RM) tmpls*
|
||||
@echo "\n Compatibility with Console :"
|
||||
echo "Hello World 1 !" | $(LZ4) | $(LZ4) -d -c
|
||||
echo "Hello World 2 !" | $(LZ4) | $(LZ4) -d | $(CAT)
|
||||
echo "Hello World 3 !" | $(LZ4) --no-frame-crc | $(LZ4) -d -c
|
||||
@echo "\n Compatibility with Append :"
|
||||
$(DATAGEN) -P100 -g1M > tmplsdg1M
|
||||
$(CAT) tmplsdg1M tmplsdg1M > tmpls2M
|
||||
$(LZ4) -B5 -v tmplsdg1M tmplsc
|
||||
$(LZ4) -d -v tmplsc tmplsr
|
||||
$(LZ4) -d -v tmplsc -c >> tmplsr
|
||||
ls -ls tmp*
|
||||
$(DIFF) tmpls2M tmplsr
|
||||
@$(RM) tmpls*
|
||||
|
||||
test-lz4-contentSize: lz4 datagen
|
||||
@echo "\n ---- test original size support ----"
|
||||
$(DATAGEN) -g15M > tmplc1
|
||||
$(LZ4) -v tmplc1 -c | $(LZ4) -t
|
||||
$(LZ4) -v --content-size tmplc1 -c | $(LZ4) -d > tmplc2
|
||||
$(DIFF) tmplc1 tmplc2
|
||||
$(LZ4) -f tmplc1 -c > tmplc1.lz4
|
||||
$(LZ4) --content-size tmplc1 -c > tmplc2.lz4
|
||||
! $(DIFF) tmplc1.lz4 tmplc2.lz4 # must differ, due to content size
|
||||
$(LZ4) --content-size < tmplc1 > tmplc3.lz4
|
||||
$(DIFF) tmplc2.lz4 tmplc3.lz4 # both must contain content size
|
||||
$(CAT) tmplc1 | $(LZ4) > tmplc4.lz4
|
||||
$(DIFF) tmplc1.lz4 tmplc4.lz4 # both don't have content size
|
||||
$(CAT) tmplc1 | $(LZ4) --content-size > tmplc5.lz4 # can't determine content size
|
||||
$(DIFF) tmplc1.lz4 tmplc5.lz4 # both don't have content size
|
||||
@$(RM) tmplc*
|
||||
|
||||
test-lz4-frame-concatenation: lz4 datagen
|
||||
@echo "\n ---- test frame concatenation ----"
|
||||
@echo -n > tmp-lfc-empty
|
||||
@echo hi > tmp-lfc-nonempty
|
||||
$(CAT) tmp-lfc-nonempty tmp-lfc-empty tmp-lfc-nonempty > tmp-lfc-src
|
||||
$(LZ4) -zq tmp-lfc-empty -c > tmp-lfc-empty.lz4
|
||||
$(LZ4) -zq tmp-lfc-nonempty -c > tmp-lfc-nonempty.lz4
|
||||
$(CAT) tmp-lfc-nonempty.lz4 tmp-lfc-empty.lz4 tmp-lfc-nonempty.lz4 > tmp-lfc-concat.lz4
|
||||
$(LZ4) -d tmp-lfc-concat.lz4 -c > tmp-lfc-result
|
||||
$(CMP) tmp-lfc-src tmp-lfc-result
|
||||
@$(RM) tmp-lfc-*
|
||||
@echo frame concatenation test completed
|
||||
|
||||
test-lz4-multiple: lz4 datagen
|
||||
@echo "\n ---- test multiple files ----"
|
||||
@$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
|
||||
@$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
|
||||
@$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
|
||||
# compress multiple files : one .lz4 per source file
|
||||
$(LZ4) -f -m tmp-tlm*
|
||||
test -f tmp-tlm1.lz4
|
||||
test -f tmp-tlm2.lz4
|
||||
test -f tmp-tlm3.lz4
|
||||
# decompress multiple files : one output file per .lz4
|
||||
mv tmp-tlm1 tmp-tlm1-orig
|
||||
mv tmp-tlm2 tmp-tlm2-orig
|
||||
mv tmp-tlm3 tmp-tlm3-orig
|
||||
$(LZ4) -d -f -m tmp-tlm*.lz4
|
||||
$(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
|
||||
$(CMP) tmp-tlm2 tmp-tlm2-orig
|
||||
$(CMP) tmp-tlm3 tmp-tlm3-orig
|
||||
# compress multiple files into stdout
|
||||
$(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
|
||||
$(RM) *.lz4
|
||||
$(LZ4) -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
|
||||
test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
|
||||
$(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
|
||||
# decompress multiple files into stdout
|
||||
$(RM) tmp-tlm-concat1 tmp-tlm-concat2
|
||||
$(LZ4) -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
|
||||
$(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
|
||||
$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
|
||||
$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
|
||||
test ! -f tmp-tlm1 # must not create file artefact
|
||||
$(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
|
||||
# compress multiple files, one of which is absent (must fail)
|
||||
! $(LZ4) -f -m tmp-tlm-concat1 notHere tmp-tlm-concat2 # must fail : notHere not present
|
||||
@$(RM) tmp-tlm*
|
||||
|
||||
test-lz4-multiple-legacy: lz4 datagen
|
||||
@echo "\n ---- test multiple files (Legacy format) ----"
|
||||
@$(DATAGEN) -s1 > tmp-tlm1 2> $(VOID)
|
||||
@$(DATAGEN) -s2 -g100K > tmp-tlm2 2> $(VOID)
|
||||
@$(DATAGEN) -s3 -g200K > tmp-tlm3 2> $(VOID)
|
||||
# compress multiple files using legacy format: one .lz4 per source file
|
||||
$(LZ4) -f -l -m tmp-tlm*
|
||||
test -f tmp-tlm1.lz4
|
||||
test -f tmp-tlm2.lz4
|
||||
test -f tmp-tlm3.lz4
|
||||
# decompress multiple files compressed using legacy format: one output file per .lz4
|
||||
mv tmp-tlm1 tmp-tlm1-orig
|
||||
mv tmp-tlm2 tmp-tlm2-orig
|
||||
mv tmp-tlm3 tmp-tlm3-orig
|
||||
$(LZ4) -d -f -m tmp-tlm*.lz4
|
||||
$(LZ4) -l -d -f -m tmp-tlm*.lz4 # -l mustn't impact -d option
|
||||
$(CMP) tmp-tlm1 tmp-tlm1-orig # must be identical
|
||||
$(CMP) tmp-tlm2 tmp-tlm2-orig
|
||||
$(CMP) tmp-tlm3 tmp-tlm3-orig
|
||||
# compress multiple files into stdout using legacy format
|
||||
$(CAT) tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 > tmp-tlm-concat1
|
||||
$(RM) *.lz4
|
||||
$(LZ4) -l -m tmp-tlm1 tmp-tlm2 tmp-tlm3 -c > tmp-tlm-concat2
|
||||
test ! -f tmp-tlm1.lz4 # must not create .lz4 artefact
|
||||
$(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
|
||||
# # # decompress multiple files into stdout using legacy format
|
||||
$(RM) tmp-tlm-concat1 tmp-tlm-concat2
|
||||
$(LZ4) -l -f -m tmp-tlm1 tmp-tlm2 tmp-tlm3 # generate .lz4 to decompress
|
||||
$(CAT) tmp-tlm1 tmp-tlm2 tmp-tlm3 > tmp-tlm-concat1 # create concatenated reference
|
||||
$(RM) tmp-tlm1 tmp-tlm2 tmp-tlm3
|
||||
$(LZ4) -d -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2
|
||||
$(LZ4) -d -l -m tmp-tlm1.lz4 tmp-tlm2.lz4 tmp-tlm3.lz4 -c > tmp-tlm-concat2 # -l mustn't impact option -d
|
||||
test ! -f tmp-tlm1 # must not create file artefact
|
||||
$(CMP) tmp-tlm-concat1 tmp-tlm-concat2 # must be equivalent
|
||||
# # # compress multiple files, one of which is absent (must fail)
|
||||
! $(LZ4) -f -l -m tmp-tlm-concat1 notHere-legacy tmp-tlm-concat2 # must fail : notHere-legacy not present
|
||||
@$(RM) tmp-tlm*
|
||||
|
||||
test-lz4-basic: lz4 datagen unlz4 lz4cat
|
||||
@echo "\n ---- test lz4 basic compression/decompression ----"
|
||||
$(DATAGEN) -g0 | $(LZ4) -v | $(LZ4) -t
|
||||
$(DATAGEN) -g16KB | $(LZ4) -9 | $(LZ4) -t
|
||||
$(DATAGEN) -g20KB > tmp-tlb-dg20k
|
||||
$(LZ4) < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
|
||||
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
|
||||
$(LZ4) --no-frame-crc < tmp-tlb-dg20k | $(LZ4) -d > tmp-tlb-dec
|
||||
$(DIFF) -q tmp-tlb-dg20k tmp-tlb-dec
|
||||
$(DATAGEN) | $(LZ4) -BI | $(LZ4) -t
|
||||
$(DATAGEN) -g6M -P99 | $(LZ4) -9BD | $(LZ4) -t
|
||||
$(DATAGEN) -g17M | $(LZ4) -9v | $(LZ4) -qt
|
||||
$(DATAGEN) -g33M | $(LZ4) --no-frame-crc | $(LZ4) -t
|
||||
$(DATAGEN) -g256MB | $(LZ4) -vqB4D | $(LZ4) -t
|
||||
@echo "hello world" > tmp-tlb-hw
|
||||
$(LZ4) --rm -f tmp-tlb-hw tmp-tlb-hw.lz4
|
||||
test ! -f tmp-tlb-hw # must fail (--rm)
|
||||
test -f tmp-tlb-hw.lz4
|
||||
$(PRGDIR)/lz4cat tmp-tlb-hw.lz4 # must display hello world
|
||||
test -f tmp-tlb-hw.lz4
|
||||
$(PRGDIR)/unlz4 --rm tmp-tlb-hw.lz4 tmp-tlb-hw
|
||||
test -f tmp-tlb-hw
|
||||
test ! -f tmp-tlb-hw.lz4 # must fail (--rm)
|
||||
test ! -f tmp-tlb-hw.lz4.lz4 # must fail (unlz4)
|
||||
$(PRGDIR)/lz4cat tmp-tlb-hw # pass-through mode
|
||||
test -f tmp-tlb-hw
|
||||
test ! -f tmp-tlb-hw.lz4 # must fail (lz4cat)
|
||||
$(LZ4) tmp-tlb-hw tmp-tlb-hw.lz4 # creates tmp-tlb-hw.lz4
|
||||
$(PRGDIR)/lz4cat < tmp-tlb-hw.lz4 > tmp-tlb3 # checks lz4cat works with stdin (#285)
|
||||
$(DIFF) -q tmp-tlb-hw tmp-tlb3
|
||||
$(PRGDIR)/lz4cat < tmp-tlb-hw > tmp-tlb2 # checks lz4cat works in pass-through mode
|
||||
$(DIFF) -q tmp-tlb-hw tmp-tlb2
|
||||
cp tmp-tlb-hw ./-d
|
||||
$(LZ4) --rm -- -d -d.lz4 # compresses ./d into ./-d.lz4
|
||||
test -f ./-d.lz4
|
||||
test ! -f ./-d
|
||||
mv ./-d.lz4 ./-z
|
||||
$(LZ4) -d --rm -- -z tmp-tlb4 # uncompresses ./-z into tmp-tlb4
|
||||
test ! -f ./-z
|
||||
$(DIFF) -q tmp-tlb-hw tmp-tlb4
|
||||
$(LZ4) -f tmp-tlb-hw
|
||||
$(LZ4) --list tmp-tlb-hw.lz4 # test --list on valid single-frame file
|
||||
$(CAT) tmp-tlb-hw >> tmp-tlb-hw.lz4
|
||||
$(LZ4) -f tmp-tlb-hw.lz4 # uncompress valid frame followed by invalid data
|
||||
$(LZ4) -BX tmp-tlb-hw -c -q | $(LZ4) -tv # test block checksum
|
||||
# $(DATAGEN) -g20KB generates the same file every single time
|
||||
# cannot save output of $(DATAGEN) -g20KB as input file to lz4 because the following shell commands are run before $(DATAGEN) -g20KB
|
||||
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast | wc -c)" -lt "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=9 | wc -c)" # -1 vs -9
|
||||
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c -1 | wc -c)" -lt "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast=1 | wc -c)" # 1 vs -1
|
||||
test "$(shell $(DATAGEN) -g20KB | $(LZ4) -c --fast=1 | wc -c)" -eq "$(shell $(DATAGEN) -g20KB| $(LZ4) -c --fast| wc -c)" # checks default fast compression is -1
|
||||
! $(LZ4) -c --fast=0 tmp-tlb-dg20K # lz4 should fail when fast=0
|
||||
! $(LZ4) -c --fast=-1 tmp-tlb-dg20K # lz4 should fail when fast=-1
|
||||
# High --fast values can result in out-of-bound dereferences #876
|
||||
$(DATAGEN) -g1M | $(LZ4) -c --fast=999999999 > /dev/null
|
||||
# Test for #596
|
||||
@echo "TEST" > tmp-tlb-test
|
||||
$(LZ4) -m tmp-tlb-test
|
||||
$(LZ4) tmp-tlb-test.lz4 tmp-tlb-test2
|
||||
$(DIFF) -q tmp-tlb-test tmp-tlb-test2
|
||||
@$(RM) tmp-tlb*
|
||||
|
||||
|
||||
|
||||
test-lz4-dict: lz4 datagen
|
||||
@echo "\n ---- test lz4 compression/decompression with dictionary ----"
|
||||
$(DATAGEN) -g16KB > tmp-dict
|
||||
$(DATAGEN) -g32KB > tmp-dict-sample-32k
|
||||
< tmp-dict-sample-32k $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-32k
|
||||
$(DATAGEN) -g128MB > tmp-dict-sample-128m
|
||||
< tmp-dict-sample-128m $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-128m
|
||||
touch tmp-dict-sample-0
|
||||
< tmp-dict-sample-0 $(LZ4) -D tmp-dict | $(LZ4) -dD tmp-dict | diff - tmp-dict-sample-0
|
||||
|
||||
< tmp-dict-sample-32k $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-32k
|
||||
< tmp-dict-sample-0 $(LZ4) -D tmp-dict-sample-0 | $(LZ4) -dD tmp-dict-sample-0 | diff - tmp-dict-sample-0
|
||||
|
||||
@echo "\n ---- test lz4 dictionary loading ----"
|
||||
$(DATAGEN) -g128KB > tmp-dict-data-128KB
|
||||
set -e; \
|
||||
for l in 0 1 4 128 32767 32768 32769 65535 65536 65537 98303 98304 98305 131071 131072 131073; do \
|
||||
$(DATAGEN) -g$$l > tmp-dict-$$l; \
|
||||
$(DD) if=tmp-dict-$$l of=tmp-dict-$$l-tail bs=1 count=65536 skip=$$((l > 65536 ? l - 65536 : 0)); \
|
||||
< tmp-dict-$$l $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l-tail | $(DIFF) - tmp-dict-data-128KB; \
|
||||
< tmp-dict-$$l-tail $(LZ4) -D stdin tmp-dict-data-128KB -c | $(LZ4) -dD tmp-dict-$$l | $(DIFF) - tmp-dict-data-128KB; \
|
||||
done
|
||||
|
||||
@$(RM) tmp-dict*
|
||||
|
||||
test-lz4-hugefile: lz4 datagen
|
||||
@echo "\n ---- test huge files compression/decompression ----"
|
||||
./datagen -g6GB | $(LZ4) -vB5D | $(LZ4) -qt
|
||||
./datagen -g4500MB | $(LZ4) -v3BD | $(LZ4) -qt
|
||||
# test large file size [2-4] GB
|
||||
@$(DATAGEN) -g3G -P100 | $(LZ4) -vv | $(LZ4) --decompress --force --sparse - tmphf1
|
||||
@ls -ls tmphf1
|
||||
@$(DATAGEN) -g3G -P100 | $(LZ4) --quiet --content-size | $(LZ4) --verbose --decompress --force --sparse - tmphf2
|
||||
@ls -ls tmphf2
|
||||
$(DIFF) -s tmphf1 tmphf2
|
||||
@$(RM) tmphf*
|
||||
|
||||
test-lz4-testmode: lz4 datagen
|
||||
@echo "\n ---- bench mode ----"
|
||||
$(LZ4) -bi0
|
||||
@echo "\n ---- test mode ----"
|
||||
! $(DATAGEN) | $(LZ4) -t
|
||||
! $(DATAGEN) | $(LZ4) -tf
|
||||
@echo "\n ---- pass-through mode ----"
|
||||
@echo "Why hello there " > tmp-tlt2.lz4
|
||||
! $(LZ4) -f tmp-tlt2.lz4 > $(VOID)
|
||||
! $(DATAGEN) | $(LZ4) -dc > $(VOID)
|
||||
! $(DATAGEN) | $(LZ4) -df > $(VOID)
|
||||
$(DATAGEN) | $(LZ4) -dcf > $(VOID)
|
||||
@echo "Hello World !" > tmp-tlt1
|
||||
$(LZ4) -dcf tmp-tlt1
|
||||
@echo "from underground..." > tmp-tlt2
|
||||
$(LZ4) -dcfm tmp-tlt1 tmp-tlt2
|
||||
@echo "\n ---- non-existing source ----"
|
||||
! $(LZ4) file-does-not-exist
|
||||
! $(LZ4) -f file-does-not-exist
|
||||
! $(LZ4) -t file-does-not-exist
|
||||
! $(LZ4) -fm file1-dne file2-dne
|
||||
@$(RM) tmp-tlt tmp-tlt1 tmp-tlt2 tmp-tlt2.lz4
|
||||
|
||||
test-lz4-opt-parser: lz4 datagen
|
||||
@echo "\n ---- test opt-parser ----"
|
||||
$(DATAGEN) -g16KB | $(LZ4) -12 | $(LZ4) -t
|
||||
$(DATAGEN) -P10 | $(LZ4) -12B4 | $(LZ4) -t
|
||||
$(DATAGEN) -g256K | $(LZ4) -12B4D | $(LZ4) -t
|
||||
$(DATAGEN) -g512K -P25 | $(LZ4) -12BD | $(LZ4) -t
|
||||
$(DATAGEN) -g1M | $(LZ4) -12B5 | $(LZ4) -t
|
||||
$(DATAGEN) -g2M -P99 | $(LZ4) -11B4D | $(LZ4) -t
|
||||
$(DATAGEN) -g4M | $(LZ4) -11vq | $(LZ4) -qt
|
||||
$(DATAGEN) -g8M | $(LZ4) -11B4 | $(LZ4) -t
|
||||
$(DATAGEN) -g16M -P90 | $(LZ4) -11B5 | $(LZ4) -t
|
||||
$(DATAGEN) -g32M -P10 | $(LZ4) -11B5D | $(LZ4) -t
|
||||
|
||||
test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-multiple-legacy \
|
||||
test-lz4-frame-concatenation test-lz4-testmode \
|
||||
test-lz4-contentSize test-lz4-dict
|
||||
@$(RM) tmp*
|
||||
|
||||
test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
|
||||
test-lz4-sparse test-lz4-hugefile test-lz4-dict
|
||||
@$(RM) tmp*
|
||||
|
||||
test-lz4c: lz4c datagen
|
||||
@echo "\n ---- test lz4c variant ----"
|
||||
$(DATAGEN) -g256MB | $(LZ4)c -l -v | $(LZ4)c -t
|
||||
|
||||
test-lz4c32: CFLAGS+=-m32
|
||||
test-lz4c32: test-lz4
|
||||
|
||||
test-interop-32-64: lz4 lz4c32 datagen
|
||||
@echo "\n ---- test interoperability 32-bits -vs- 64 bits ----"
|
||||
$(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4) -t
|
||||
$(DATAGEN) -P10 | $(LZ4) -9B4 | $(LZ4)c32 -t
|
||||
$(DATAGEN) | $(LZ4)c32 | $(LZ4) -t
|
||||
$(DATAGEN) -g1M | $(LZ4) -3B5 | $(LZ4)c32 -t
|
||||
$(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4) -qt
|
||||
$(DATAGEN) -g1G -P90 | $(LZ4) | $(LZ4)c32 -t
|
||||
$(DATAGEN) -g6GB | $(LZ4)c32 -vq9BD | $(LZ4) -qt
|
||||
|
||||
test-lz4c32-basic: lz4c32 datagen
|
||||
@echo "\n ---- test lz4c32 32-bits version ----"
|
||||
$(DATAGEN) -g16KB | $(LZ4)c32 -9 | $(LZ4)c32 -t
|
||||
$(DATAGEN) | $(LZ4)c32 | $(LZ4)c32 -t
|
||||
$(DATAGEN) -g256MB | $(LZ4)c32 -vqB4D | $(LZ4)c32 -qt
|
||||
$(DATAGEN) -g6GB | $(LZ4)c32 -vqB5D | $(LZ4)c32 -qt
|
||||
|
||||
test-platform:
|
||||
@echo "\n ---- test lz4 $(QEMU_SYS) platform ----"
|
||||
$(QEMU_SYS) $(DATAGEN) -g16KB | $(QEMU_SYS) $(LZ4) -9 | $(QEMU_SYS) $(LZ4) -t
|
||||
$(QEMU_SYS) $(DATAGEN) | $(QEMU_SYS) $(LZ4) | $(QEMU_SYS) $(LZ4) -t
|
||||
$(QEMU_SYS) $(DATAGEN) -g256MB | $(QEMU_SYS) $(LZ4) -vqB4D | $(QEMU_SYS) $(LZ4) -qt
|
||||
ifneq ($(QEMU_SYS),qemu-arm-static)
|
||||
$(QEMU_SYS) $(DATAGEN) -g3GB | $(QEMU_SYS) $(LZ4) -vqB5D | $(QEMU_SYS) $(LZ4) -qt
|
||||
endif
|
||||
|
||||
test-fullbench: fullbench
|
||||
./fullbench --no-prompt $(NB_LOOPS) $(TEST_FILES)
|
||||
|
||||
test-fullbench32: CFLAGS += -m32
|
||||
test-fullbench32: test-fullbench
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer $(FUZZER_TIME)
|
||||
|
||||
test-fuzzer32: CFLAGS += -m32
|
||||
test-fuzzer32: test-fuzzer
|
||||
|
||||
test-frametest: frametest
|
||||
./frametest -v $(FUZZER_TIME)
|
||||
|
||||
test-frametest32: CFLAGS += -m32
|
||||
test-frametest32: test-frametest
|
||||
|
||||
test-mem: lz4 datagen fuzzer frametest fullbench
|
||||
@echo "\n ---- valgrind tests : memory analyzer ----"
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(DATAGEN) -g50M > $(VOID)
|
||||
$(DATAGEN) -g16KB > ftmdg16K
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -BD -f ftmdg16K $(VOID)
|
||||
$(DATAGEN) -g16KB -s2 > ftmdg16K2
|
||||
$(DATAGEN) -g16KB -s3 > ftmdg16K3
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --force --multiple ftmdg16K ftmdg16K2 ftmdg16K3
|
||||
$(DATAGEN) -g7MB > ftmdg7M
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -9 -B5D -f ftmdg7M ftmdg16K2
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -t ftmdg16K2
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -bi1 ftmdg7M
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fullbench -i1 ftmdg7M ftmdg16K2
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) -B4D -f -vq ftmdg7M $(VOID)
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m ftm*.lz4
|
||||
valgrind --leak-check=yes --error-exitcode=1 $(LZ4) --list -m -v ftm*.lz4
|
||||
$(RM) ftm*
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./fuzzer -i64 -t1
|
||||
valgrind --leak-check=yes --error-exitcode=1 ./frametest -i256
|
||||
|
||||
test-mem32: lz4c32 datagen
|
||||
# unfortunately, valgrind doesn't seem to work with non-native binary...
|
||||
|
||||
test-decompress-partial : decompress-partial
|
||||
@echo "\n ---- test decompress-partial ----"
|
||||
./decompress-partial$(EXT)
|
||||
|
||||
endif
|
||||
71
lz4/tests/README.md
Normal file
71
lz4/tests/README.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
Programs and scripts for automated testing of LZ4
|
||||
=======================================================
|
||||
|
||||
This directory contains the following programs and scripts:
|
||||
- `datagen` : Synthetic and parametrable data generator, for tests
|
||||
- `frametest` : Test tool that checks lz4frame integrity on target platform
|
||||
- `fullbench` : Precisely measure speed for each lz4 inner functions
|
||||
- `fuzzer` : Test tool, to check lz4 integrity on target platform
|
||||
- `test-lz4-speed.py` : script for testing lz4 speed difference between commits
|
||||
- `test-lz4-versions.py` : compatibility test between lz4 versions stored on Github
|
||||
|
||||
|
||||
#### `test-lz4-versions.py` - script for testing lz4 interoperability between versions
|
||||
|
||||
This script creates `versionsTest` directory to which lz4 repository is cloned.
|
||||
Then all taged (released) versions of lz4 are compiled.
|
||||
In the following step interoperability between lz4 versions is checked.
|
||||
|
||||
|
||||
#### `test-lz4-speed.py` - script for testing lz4 speed difference between commits
|
||||
|
||||
This script creates `speedTest` directory to which lz4 repository is cloned.
|
||||
Then it compiles all branches of lz4 and performs a speed benchmark for a given list of files (the `testFileNames` parameter).
|
||||
After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits.
|
||||
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
|
||||
The results of the speed benchmark are compared to the previous results.
|
||||
If compression or decompression speed for one of lz4 levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
|
||||
If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter).
|
||||
|
||||
Additional remarks:
|
||||
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
|
||||
- Using the script with virtual machines can lead to large variations of speed results
|
||||
- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75)
|
||||
- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning
|
||||
|
||||
|
||||
The example usage with two test files, one e-mail address, and with an additional message:
|
||||
```
|
||||
./test-lz4-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
|
||||
```
|
||||
|
||||
To run the script in background please use:
|
||||
```
|
||||
nohup ./test-lz4-speed.py testFileNames emails &
|
||||
```
|
||||
|
||||
The full list of parameters:
|
||||
```
|
||||
positional arguments:
|
||||
testFileNames file names list for speed benchmark
|
||||
emails list of e-mail addresses to send warnings
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--message MESSAGE attach an additional message to e-mail
|
||||
--lowerLimit LOWERLIMIT
|
||||
send email if speed is lower than given limit
|
||||
--maxLoadAvg MAXLOADAVG
|
||||
maximum load average to start testing
|
||||
--lastCLevel LASTCLEVEL
|
||||
last compression level for testing
|
||||
--sleepTime SLEEPTIME
|
||||
frequency of repository checking in seconds
|
||||
```
|
||||
|
||||
|
||||
#### License
|
||||
|
||||
All files in this directory are licensed under GPL-v2.
|
||||
See [COPYING](COPYING) for details.
|
||||
The text of the license is also included at the top of each source file.
|
||||
303
lz4/tests/checkFrame.c
Normal file
303
lz4/tests/checkFrame.c
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
/*
|
||||
checkFrame - verify frame headers
|
||||
Copyright (C) Yann Collet 2014-present
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://www.lz4.org
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
*/
|
||||
|
||||
/*-************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "util.h" /* U32 */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
|
||||
#include <assert.h>
|
||||
#include "lz4frame.h" /* include multiple times to test correctness/safety */
|
||||
#include "lz4frame.h"
|
||||
#define LZ4F_STATIC_LINKING_ONLY
|
||||
#include "lz4frame.h"
|
||||
#include "lz4frame.h"
|
||||
#include "lz4.h" /* LZ4_VERSION_STRING */
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#include "xxhash.h" /* XXH64 */
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1U<<10)
|
||||
#define MB *(1U<<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
|
||||
/**************************************
|
||||
* Exceptions
|
||||
***************************************/
|
||||
#ifndef DEBUG
|
||||
# define DEBUG 0
|
||||
#endif
|
||||
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
|
||||
#define EXM_THROW(error, ...) \
|
||||
{ \
|
||||
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
|
||||
DISPLAYLEVEL(1, "Error %i : ", error); \
|
||||
DISPLAYLEVEL(1, __VA_ARGS__); \
|
||||
DISPLAYLEVEL(1, " \n"); \
|
||||
return(error); \
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*-***************************************
|
||||
* Local Parameters
|
||||
*****************************************/
|
||||
static U32 no_prompt = 0;
|
||||
static U32 displayLevel = 2;
|
||||
static U32 use_pause = 0;
|
||||
|
||||
|
||||
/*-*******************************************************
|
||||
* Fuzzer functions
|
||||
*********************************************************/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
#define MAX(a,b) ( (a) > (b) ? (a) : (b) )
|
||||
|
||||
typedef struct {
|
||||
void* srcBuffer;
|
||||
size_t srcBufferSize;
|
||||
void* dstBuffer;
|
||||
size_t dstBufferSize;
|
||||
LZ4F_decompressionContext_t ctx;
|
||||
} cRess_t;
|
||||
|
||||
static int createCResources(cRess_t* ress)
|
||||
{
|
||||
ress->srcBufferSize = 4 MB;
|
||||
ress->srcBuffer = malloc(ress->srcBufferSize);
|
||||
ress->dstBufferSize = 4 MB;
|
||||
ress->dstBuffer = malloc(ress->dstBufferSize);
|
||||
|
||||
if (!ress->srcBuffer || !ress->dstBuffer) {
|
||||
free(ress->srcBuffer);
|
||||
free(ress->dstBuffer);
|
||||
EXM_THROW(20, "Allocation error : not enough memory");
|
||||
}
|
||||
|
||||
if (LZ4F_isError( LZ4F_createDecompressionContext(&(ress->ctx), LZ4F_VERSION) )) {
|
||||
free(ress->srcBuffer);
|
||||
free(ress->dstBuffer);
|
||||
EXM_THROW(21, "Unable to create decompression context");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void freeCResources(cRess_t ress)
|
||||
{
|
||||
free(ress.srcBuffer);
|
||||
free(ress.dstBuffer);
|
||||
|
||||
(void) LZ4F_freeDecompressionContext(ress.ctx);
|
||||
}
|
||||
|
||||
int frameCheck(cRess_t ress, FILE* const srcFile, unsigned bsid, size_t blockSize)
|
||||
{
|
||||
LZ4F_errorCode_t nextToLoad = 0;
|
||||
size_t curblocksize = 0;
|
||||
int partialBlock = 0;
|
||||
|
||||
/* Main Loop */
|
||||
for (;;) {
|
||||
size_t readSize;
|
||||
size_t pos = 0;
|
||||
size_t decodedBytes = ress.dstBufferSize;
|
||||
size_t remaining;
|
||||
LZ4F_frameInfo_t frameInfo;
|
||||
|
||||
/* Read input */
|
||||
readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile);
|
||||
if (!readSize) break; /* reached end of file or stream */
|
||||
|
||||
while (pos < readSize) { /* still to read */
|
||||
/* Decode Input (at least partially) */
|
||||
if (!nextToLoad) {
|
||||
/* LZ4F_decompress returned 0 : starting new frame */
|
||||
curblocksize = 0;
|
||||
remaining = readSize - pos;
|
||||
nextToLoad = LZ4F_getFrameInfo(ress.ctx, &frameInfo, (char*)(ress.srcBuffer)+pos, &remaining);
|
||||
if (LZ4F_isError(nextToLoad))
|
||||
EXM_THROW(22, "Error getting frame info: %s",
|
||||
LZ4F_getErrorName(nextToLoad));
|
||||
if (frameInfo.blockSizeID != bsid)
|
||||
EXM_THROW(23, "Block size ID %u != expected %u",
|
||||
frameInfo.blockSizeID, bsid);
|
||||
pos += remaining;
|
||||
/* nextToLoad should be block header size */
|
||||
remaining = nextToLoad;
|
||||
decodedBytes = ress.dstBufferSize;
|
||||
nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
|
||||
if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
|
||||
pos += remaining;
|
||||
}
|
||||
decodedBytes = ress.dstBufferSize;
|
||||
/* nextToLoad should be just enough to cover the next block */
|
||||
if (nextToLoad > (readSize - pos)) {
|
||||
/* block is not fully contained in current buffer */
|
||||
partialBlock = 1;
|
||||
remaining = readSize - pos;
|
||||
} else {
|
||||
if (partialBlock) {
|
||||
partialBlock = 0;
|
||||
}
|
||||
remaining = nextToLoad;
|
||||
}
|
||||
nextToLoad = LZ4F_decompress(ress.ctx, ress.dstBuffer, &decodedBytes, (char*)(ress.srcBuffer)+pos, &remaining, NULL);
|
||||
if (LZ4F_isError(nextToLoad)) EXM_THROW(24, "Decompression error : %s", LZ4F_getErrorName(nextToLoad));
|
||||
curblocksize += decodedBytes;
|
||||
pos += remaining;
|
||||
if (!partialBlock) {
|
||||
/* detect small block due to end of frame; the final 4-byte frame checksum could be left in the buffer */
|
||||
if ((curblocksize != 0) && (nextToLoad > 4)) {
|
||||
if (curblocksize != blockSize)
|
||||
EXM_THROW(25, "Block size %u != expected %u, pos %u\n",
|
||||
(unsigned)curblocksize, (unsigned)blockSize, (unsigned)pos);
|
||||
}
|
||||
curblocksize = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* can be out because readSize == 0, which could be an fread() error */
|
||||
if (ferror(srcFile)) EXM_THROW(26, "Read error");
|
||||
|
||||
if (nextToLoad!=0) EXM_THROW(27, "Unfinished stream");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int FUZ_usage(const char* programName)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [args] filename\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -b# : expected blocksizeID [4-7] (required)\n");
|
||||
DISPLAY( " -B# : expected blocksize [32-4194304] (required)\n");
|
||||
DISPLAY( " -v : verbose\n");
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int argNb;
|
||||
unsigned bsid=0;
|
||||
size_t blockSize=0;
|
||||
const char* const programName = argv[0];
|
||||
|
||||
/* Check command line */
|
||||
for (argNb=1; argNb<argc; argNb++) {
|
||||
const char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Decode command (note : aggregated short commands are allowed) */
|
||||
if (argument[0]=='-') {
|
||||
if (!strcmp(argument, "--no-prompt")) {
|
||||
no_prompt=1;
|
||||
displayLevel=1;
|
||||
continue;
|
||||
}
|
||||
argument++;
|
||||
|
||||
while (*argument!=0) {
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return FUZ_usage(programName);
|
||||
case 'v':
|
||||
argument++;
|
||||
displayLevel++;
|
||||
break;
|
||||
case 'q':
|
||||
argument++;
|
||||
displayLevel--;
|
||||
break;
|
||||
case 'p': /* pause at the end */
|
||||
argument++;
|
||||
use_pause = 1;
|
||||
break;
|
||||
|
||||
case 'b':
|
||||
argument++;
|
||||
bsid=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
bsid *= 10;
|
||||
bsid += (unsigned)(*argument - '0');
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'B':
|
||||
argument++;
|
||||
blockSize=0;
|
||||
while ((*argument>='0') && (*argument<='9')) {
|
||||
blockSize *= 10;
|
||||
blockSize += (size_t)(*argument - '0');
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
;
|
||||
return FUZ_usage(programName);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
int err;
|
||||
FILE *srcFile;
|
||||
cRess_t ress;
|
||||
if (bsid == 0 || blockSize == 0)
|
||||
return FUZ_usage(programName);
|
||||
DISPLAY("Starting frame checker (%i-bits, %s)\n", (int)(sizeof(size_t)*8), LZ4_VERSION_STRING);
|
||||
err = createCResources(&ress);
|
||||
if (err) return (err);
|
||||
srcFile = fopen(argument, "rb");
|
||||
if ( srcFile==NULL ) {
|
||||
freeCResources(ress);
|
||||
EXM_THROW(1, "%s: %s \n", argument, strerror(errno));
|
||||
}
|
||||
assert (srcFile != NULL);
|
||||
err = frameCheck(ress, srcFile, bsid, blockSize);
|
||||
freeCResources(ress);
|
||||
fclose(srcFile);
|
||||
return (err);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
79
lz4/tests/checkTag.c
Normal file
79
lz4/tests/checkTag.c
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
checkTag.c - Version validation tool for LZ4
|
||||
Copyright (C) Yann Collet 2018 - present
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 homepage : http://www.lz4.org
|
||||
- LZ4 source repo : https://github.com/lz4/lz4
|
||||
*/
|
||||
|
||||
/* checkTag command :
|
||||
* $ ./checkTag tag
|
||||
* checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
|
||||
* The tag is then compared to LZ4 version number.
|
||||
* They are compatible if first 3 digits are identical.
|
||||
* Anything beyond that is free, and doesn't impact validation.
|
||||
* Example : tag v1.8.1.2 is compatible with version 1.8.1
|
||||
* When tag and version are not compatible, program exits with error code 1.
|
||||
* When they are compatible, it exists with a code 0.
|
||||
* checkTag is intended to be used in automated testing environment.
|
||||
*/
|
||||
|
||||
#include <stdio.h> /* printf */
|
||||
#include <string.h> /* strlen, strncmp */
|
||||
#include "lz4.h" /* LZ4_VERSION_STRING */
|
||||
|
||||
|
||||
/* validate() :
|
||||
* @return 1 if tag is compatible, 0 if not.
|
||||
*/
|
||||
static int validate(const char* const tag)
|
||||
{
|
||||
size_t const tagLength = strlen(tag);
|
||||
size_t const verLength = strlen(LZ4_VERSION_STRING);
|
||||
|
||||
if (tagLength < 2) return 0;
|
||||
if (tag[0] != 'v') return 0;
|
||||
if (tagLength <= verLength) return 0;
|
||||
|
||||
if (strncmp(LZ4_VERSION_STRING, tag+1, verLength)) return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
const char* const tag = argv[1];
|
||||
if (argc!=2) {
|
||||
printf("incorrect usage : %s tag \n", exeName);
|
||||
return 2;
|
||||
}
|
||||
|
||||
printf("Version : %s \n", LZ4_VERSION_STRING);
|
||||
printf("Tag : %s \n", tag);
|
||||
|
||||
if (validate(tag)) {
|
||||
printf("OK : tag is compatible with lz4 version \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
printf("!! error : tag and versions are not compatible !! \n");
|
||||
return 1;
|
||||
}
|
||||
172
lz4/tests/datagencli.c
Normal file
172
lz4/tests/datagencli.c
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
datagencli.c
|
||||
compressible data command line generator
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- Public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "util.h" /* U32 */
|
||||
#include <stdio.h> /* fprintf, stderr */
|
||||
#include "datagen.h" /* RDG_generate */
|
||||
#include "lz4.h" /* LZ4_VERSION_STRING */
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define SIZE_DEFAULT (64 KB)
|
||||
#define SEED_DEFAULT 0
|
||||
#define COMPRESSIBILITY_DEFAULT 50
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
|
||||
static unsigned displayLevel = 2;
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Command line
|
||||
*********************************************************/
|
||||
static int usage(char* programName)
|
||||
{
|
||||
DISPLAY( "Compressible data generator\n");
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [size] [args]\n", programName);
|
||||
DISPLAY( "\n");
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT);
|
||||
DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT);
|
||||
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT);
|
||||
DISPLAY( " -h : display help and exit\n");
|
||||
DISPLAY( "Special values :\n");
|
||||
DISPLAY( " -P0 : generate incompressible noise\n");
|
||||
DISPLAY( " -P100 : generate sparse files\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
double proba = (double)COMPRESSIBILITY_DEFAULT / 100;
|
||||
double litProba = 0.0;
|
||||
U64 size = SIZE_DEFAULT;
|
||||
U32 seed = SEED_DEFAULT;
|
||||
char* programName;
|
||||
|
||||
/* Check command line */
|
||||
programName = argv[0];
|
||||
for(argNb=1; argNb<argc; argNb++)
|
||||
{
|
||||
char* argument = argv[argNb];
|
||||
|
||||
if(!argument) continue; /* Protection if argument empty */
|
||||
|
||||
/* Handle commands. Aggregated commands are allowed */
|
||||
if (*argument=='-')
|
||||
{
|
||||
argument++;
|
||||
while (*argument!=0)
|
||||
{
|
||||
switch(*argument)
|
||||
{
|
||||
case 'h':
|
||||
return usage(programName);
|
||||
case 'g':
|
||||
argument++;
|
||||
size=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
size *= 10;
|
||||
size += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (*argument=='K') { size <<= 10; argument++; }
|
||||
if (*argument=='M') { size <<= 20; argument++; }
|
||||
if (*argument=='G') { size <<= 30; argument++; }
|
||||
if (*argument=='B') { argument++; }
|
||||
break;
|
||||
case 's':
|
||||
argument++;
|
||||
seed=0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
seed *= 10;
|
||||
seed += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
proba=0.0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba>100.) proba=100.;
|
||||
proba /= 100.;
|
||||
break;
|
||||
case 'L': /* hidden argument : Literal distribution probability */
|
||||
argument++;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
litProba *= 10;
|
||||
litProba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
argument++;
|
||||
break;
|
||||
default:
|
||||
return usage(programName);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", LZ4_VERSION_STRING);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
|
||||
|
||||
RDG_genOut(size, proba, litProba, seed);
|
||||
DISPLAYLEVEL(1, "\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
49
lz4/tests/decompress-partial.c
Normal file
49
lz4/tests/decompress-partial.c
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#include "stdio.h"
|
||||
#include "string.h"
|
||||
#include "lz4.h"
|
||||
|
||||
const char source[] =
|
||||
"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n"
|
||||
"tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n"
|
||||
"veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n"
|
||||
"commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n"
|
||||
"velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n"
|
||||
"cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n"
|
||||
"est laborum.\n"
|
||||
"\n"
|
||||
"Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n"
|
||||
"doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n"
|
||||
"veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n"
|
||||
"ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n"
|
||||
"consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n"
|
||||
"porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n"
|
||||
"adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n"
|
||||
"et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n"
|
||||
"nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n"
|
||||
"ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n"
|
||||
"voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n"
|
||||
"dolorem eum fugiat quo voluptas nulla pariatur?\n";
|
||||
|
||||
#define BUFFER_SIZE 2048
|
||||
|
||||
int main(void)
|
||||
{
|
||||
int srcLen = (int)strlen(source);
|
||||
char cmpBuffer[BUFFER_SIZE];
|
||||
char outBuffer[BUFFER_SIZE];
|
||||
int cmpSize;
|
||||
int i;
|
||||
|
||||
cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE);
|
||||
|
||||
for (i = cmpSize; i < cmpSize + 10; ++i) {
|
||||
int result = LZ4_decompress_safe_partial(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE);
|
||||
if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) {
|
||||
printf("test decompress-partial error \n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
printf("test decompress-partial OK \n");
|
||||
return 0;
|
||||
}
|
||||
1281
lz4/tests/frametest.c
Normal file
1281
lz4/tests/frametest.c
Normal file
File diff suppressed because it is too large
Load diff
869
lz4/tests/fullbench.c
Normal file
869
lz4/tests/fullbench.c
Normal file
|
|
@ -0,0 +1,869 @@
|
|||
/*
|
||||
bench.c - Demo program to benchmark open-source compression algorithm
|
||||
Copyright (C) Yann Collet 2012-2016
|
||||
|
||||
GPL v2 License
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
You can contact the author at :
|
||||
- LZ4 source repository : https://github.com/lz4/lz4
|
||||
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||
*/
|
||||
|
||||
|
||||
#if defined(_MSC_VER) || defined(_WIN32)
|
||||
/* S_ISREG & gettimeofday() are not supported by MSVC */
|
||||
# define BMK_LEGACY_TIMER 1
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Includes
|
||||
**************************************/
|
||||
#include "platform.h" /* _CRT_SECURE_NO_WARNINGS, Large Files support */
|
||||
#include "util.h" /* U32, UTIL_getFileSize */
|
||||
#include <stdlib.h> /* malloc, free */
|
||||
#include <stdio.h> /* fprintf, fopen, ftello */
|
||||
#include <sys/types.h> /* stat64 */
|
||||
#include <sys/stat.h> /* stat64 */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <time.h> /* clock_t, clock(), CLOCKS_PER_SEC */
|
||||
|
||||
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* LZ4_decompress_fast */
|
||||
#include "lz4.h"
|
||||
#include "lz4hc.h"
|
||||
#include "lz4frame.h"
|
||||
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
#define PROGRAM_DESCRIPTION "LZ4 speed analyzer"
|
||||
#define AUTHOR "Yann Collet"
|
||||
#define WELCOME_MESSAGE "*** %s v%s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, LZ4_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR
|
||||
|
||||
#define NBLOOPS 6
|
||||
#define TIMELOOP (CLOCKS_PER_SEC * 25 / 10)
|
||||
|
||||
#define KB *(1 <<10)
|
||||
#define MB *(1 <<20)
|
||||
#define GB *(1U<<30)
|
||||
|
||||
#define KNUTH 2654435761U
|
||||
#define MAX_MEM (1920 MB)
|
||||
#define DEFAULT_CHUNKSIZE (4 MB)
|
||||
|
||||
#define ALL_COMPRESSORS 0
|
||||
#define ALL_DECOMPRESSORS 0
|
||||
|
||||
|
||||
/**************************************
|
||||
* Local structures
|
||||
**************************************/
|
||||
struct chunkParameters
|
||||
{
|
||||
U32 id;
|
||||
char* origBuffer;
|
||||
char* compressedBuffer;
|
||||
int origSize;
|
||||
int compressedSize;
|
||||
};
|
||||
|
||||
|
||||
/**************************************
|
||||
* Macros
|
||||
**************************************/
|
||||
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
|
||||
#define PROGRESS(...) g_noPrompt ? 0 : DISPLAY(__VA_ARGS__)
|
||||
|
||||
|
||||
/**************************************
|
||||
* Benchmark Parameters
|
||||
**************************************/
|
||||
static int g_chunkSize = DEFAULT_CHUNKSIZE;
|
||||
static int g_nbIterations = NBLOOPS;
|
||||
static int g_pause = 0;
|
||||
static int g_compressionTest = 1;
|
||||
static int g_compressionAlgo = ALL_COMPRESSORS;
|
||||
static int g_decompressionTest = 1;
|
||||
static int g_decompressionAlgo = ALL_DECOMPRESSORS;
|
||||
static int g_noPrompt = 0;
|
||||
|
||||
static void BMK_setBlocksize(int bsize)
|
||||
{
|
||||
g_chunkSize = bsize;
|
||||
DISPLAY("-Using Block Size of %i KB-\n", g_chunkSize>>10);
|
||||
}
|
||||
|
||||
static void BMK_setNbIterations(int nbLoops)
|
||||
{
|
||||
g_nbIterations = nbLoops;
|
||||
DISPLAY("- %i iterations -\n", g_nbIterations);
|
||||
}
|
||||
|
||||
static void BMK_setPause(void)
|
||||
{
|
||||
g_pause = 1;
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Private functions
|
||||
*********************************************************/
|
||||
static clock_t BMK_GetClockSpan( clock_t clockStart )
|
||||
{
|
||||
return clock() - clockStart; /* works even if overflow; max span ~30 mn */
|
||||
}
|
||||
|
||||
|
||||
static size_t BMK_findMaxMem(U64 requiredMem)
|
||||
{
|
||||
size_t step = 64 MB;
|
||||
BYTE* testmem = NULL;
|
||||
|
||||
requiredMem = (((requiredMem >> 26) + 1) << 26);
|
||||
requiredMem += 2*step;
|
||||
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
|
||||
|
||||
while (!testmem) {
|
||||
if (requiredMem > step) requiredMem -= step;
|
||||
else requiredMem >>= 1;
|
||||
testmem = (BYTE*) malloc ((size_t)requiredMem);
|
||||
}
|
||||
free (testmem);
|
||||
|
||||
/* keep some space available */
|
||||
if (requiredMem > step) requiredMem -= step;
|
||||
else requiredMem >>= 1;
|
||||
|
||||
return (size_t)requiredMem;
|
||||
}
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Memory management, to test LZ4_USER_MEMORY_FUNCTIONS
|
||||
*********************************************************/
|
||||
void* LZ4_malloc(size_t s) { return malloc(s); }
|
||||
void* LZ4_calloc(size_t n, size_t s) { return calloc(n,s); }
|
||||
void LZ4_free(void* p) { free(p); }
|
||||
|
||||
|
||||
/*********************************************************
|
||||
* Benchmark function
|
||||
*********************************************************/
|
||||
static LZ4_stream_t LZ4_stream;
|
||||
static void local_LZ4_resetDictT(void)
|
||||
{
|
||||
void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
|
||||
assert(r != NULL); (void)r;
|
||||
}
|
||||
|
||||
static void local_LZ4_createStream(void)
|
||||
{
|
||||
void* const r = LZ4_initStream(&LZ4_stream, sizeof(LZ4_stream));
|
||||
assert(r != NULL); (void)r;
|
||||
}
|
||||
|
||||
static int local_LZ4_saveDict(const char* in, char* out, int inSize)
|
||||
{
|
||||
(void)in;
|
||||
return LZ4_saveDict(&LZ4_stream, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_default_large(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_default_small(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_default(in, out, inSize, LZ4_compressBound(inSize)-1);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_destSize(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_destSize(in, out, &inSize, LZ4_compressBound(inSize)-1);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast0(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 0);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast1(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 1);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast2(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 2);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast17(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast(in, out, inSize, LZ4_compressBound(inSize), 17);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast_extState0(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast_extState(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_fast_continue0(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_fast_continue(&LZ4_stream, in, out, inSize, LZ4_compressBound(inSize), 0);
|
||||
}
|
||||
|
||||
#ifndef LZ4_DLL_IMPORT
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* declare hidden function */
|
||||
extern int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
static int local_LZ4_compress_forceDict(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_forceExtDict(&LZ4_stream, in, out, inSize);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/* HC compression functions */
|
||||
LZ4_streamHC_t LZ4_streamHC;
|
||||
static void local_LZ4_resetStreamHC(void)
|
||||
{
|
||||
LZ4_initStreamHC(&LZ4_streamHC, sizeof(LZ4_streamHC));
|
||||
}
|
||||
|
||||
static int local_LZ4_saveDictHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
(void)in;
|
||||
return LZ4_saveDictHC(&LZ4_streamHC, out, inSize);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_HC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_HC(in, out, inSize, LZ4_compressBound(inSize), 9);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_HC_extStateHC(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_HC_extStateHC(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize), 9);
|
||||
}
|
||||
|
||||
static int local_LZ4_compress_HC_continue(const char* in, char* out, int inSize)
|
||||
{
|
||||
return LZ4_compress_HC_continue(&LZ4_streamHC, in, out, inSize, LZ4_compressBound(inSize));
|
||||
}
|
||||
|
||||
|
||||
/* decompression functions */
|
||||
static int local_LZ4_decompress_fast(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast(in, out, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_fast_usingDict_prefix(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_fast_usingExtDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_fast_usingDict(in, out, outSize, out - 65536, 65535);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_withPrefix64k(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
LZ4_decompress_safe_withPrefix64k(in, out, inSize, outSize);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
static int local_LZ4_decompress_safe_usingDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
LZ4_decompress_safe_usingDict(in, out, inSize, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
|
||||
#ifndef LZ4_DLL_IMPORT
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
extern int LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize, const void* dict, size_t dictSize);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
static int local_LZ4_decompress_safe_forceExtDict(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
(void)inSize;
|
||||
LZ4_decompress_safe_forceExtDict(in, out, inSize, outSize, out - 65536, 65536);
|
||||
return outSize;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int local_LZ4_decompress_safe_partial(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
int result = LZ4_decompress_safe_partial(in, out, inSize, outSize - 5, outSize);
|
||||
if (result < 0) return result;
|
||||
return outSize;
|
||||
}
|
||||
|
||||
|
||||
/* frame functions */
|
||||
static int local_LZ4F_compressFrame(const char* in, char* out, int inSize)
|
||||
{
|
||||
assert(inSize >= 0);
|
||||
return (int)LZ4F_compressFrame(out, LZ4F_compressFrameBound((size_t)inSize, NULL), in, (size_t)inSize, NULL);
|
||||
}
|
||||
|
||||
static LZ4F_decompressionContext_t g_dCtx;
|
||||
|
||||
static int local_LZ4F_decompress(const char* in, char* out, int inSize, int outSize)
|
||||
{
|
||||
size_t srcSize = (size_t)inSize;
|
||||
size_t dstSize = (size_t)outSize;
|
||||
size_t result;
|
||||
assert(inSize >= 0);
|
||||
assert(outSize >= 0);
|
||||
result = LZ4F_decompress(g_dCtx, out, &dstSize, in, &srcSize, NULL);
|
||||
if (result!=0) { DISPLAY("Error decompressing frame : unfinished frame \n"); exit(8); }
|
||||
if (srcSize != (size_t)inSize) { DISPLAY("Error decompressing frame : read size incorrect \n"); exit(9); }
|
||||
return (int)dstSize;
|
||||
}
|
||||
|
||||
static int local_LZ4F_decompress_followHint(const char* src, char* dst, int srcSize, int dstSize)
|
||||
{
|
||||
size_t totalInSize = (size_t)srcSize;
|
||||
size_t maxOutSize = (size_t)dstSize;
|
||||
|
||||
size_t inPos = 0;
|
||||
size_t inSize = 0;
|
||||
size_t outPos = 0;
|
||||
size_t outRemaining = maxOutSize - outPos;
|
||||
|
||||
for (;;) {
|
||||
size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
|
||||
assert(!LZ4F_isError(sizeHint));
|
||||
|
||||
inPos += inSize;
|
||||
inSize = sizeHint;
|
||||
|
||||
outPos += outRemaining;
|
||||
outRemaining = maxOutSize - outPos;
|
||||
|
||||
if (!sizeHint) break;
|
||||
}
|
||||
|
||||
/* frame completed */
|
||||
if (inPos != totalInSize) {
|
||||
DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
|
||||
(unsigned)inPos, (unsigned)totalInSize);
|
||||
exit(10);
|
||||
}
|
||||
return (int)outPos;
|
||||
|
||||
}
|
||||
|
||||
/* always provide input by block of 64 KB */
|
||||
static int local_LZ4F_decompress_noHint(const char* src, char* dst, int srcSize, int dstSize)
|
||||
{
|
||||
size_t totalInSize = (size_t)srcSize;
|
||||
size_t maxOutSize = (size_t)dstSize;
|
||||
|
||||
size_t inPos = 0;
|
||||
size_t inSize = 64 KB;
|
||||
size_t outPos = 0;
|
||||
size_t outRemaining = maxOutSize - outPos;
|
||||
|
||||
for (;;) {
|
||||
size_t const sizeHint = LZ4F_decompress(g_dCtx, dst+outPos, &outRemaining, src+inPos, &inSize, NULL);
|
||||
assert(!LZ4F_isError(sizeHint));
|
||||
|
||||
inPos += inSize;
|
||||
inSize = (inPos + 64 KB <= totalInSize) ? 64 KB : totalInSize - inPos;
|
||||
|
||||
outPos += outRemaining;
|
||||
outRemaining = maxOutSize - outPos;
|
||||
|
||||
if (!sizeHint) break;
|
||||
}
|
||||
|
||||
/* frame completed */
|
||||
if (inPos != totalInSize) {
|
||||
DISPLAY("Error decompressing frame : must read (%u) full frame (%u) \n",
|
||||
(unsigned)inPos, (unsigned)totalInSize);
|
||||
exit(10);
|
||||
}
|
||||
return (int)outPos;
|
||||
|
||||
}
|
||||
|
||||
#define NB_COMPRESSION_ALGORITHMS 100
|
||||
#define NB_DECOMPRESSION_ALGORITHMS 100
|
||||
int fullSpeedBench(const char** fileNamesTable, int nbFiles)
|
||||
{
|
||||
int fileIdx=0;
|
||||
|
||||
/* Init */
|
||||
{ size_t const errorCode = LZ4F_createDecompressionContext(&g_dCtx, LZ4F_VERSION);
|
||||
if (LZ4F_isError(errorCode)) { DISPLAY("dctx allocation issue \n"); return 10; } }
|
||||
|
||||
/* Loop for each fileName */
|
||||
while (fileIdx<nbFiles) {
|
||||
char* orig_buff = NULL;
|
||||
struct chunkParameters* chunkP = NULL;
|
||||
char* compressed_buff=NULL;
|
||||
const char* const inFileName = fileNamesTable[fileIdx++];
|
||||
FILE* const inFile = fopen( inFileName, "rb" );
|
||||
U64 const inFileSize = UTIL_getFileSize(inFileName);
|
||||
size_t benchedSize = BMK_findMaxMem(inFileSize*2) / 2; /* because 2 buffers */
|
||||
int nbChunks;
|
||||
int maxCompressedChunkSize;
|
||||
size_t readSize;
|
||||
int compressedBuffSize;
|
||||
U32 crcOriginal;
|
||||
|
||||
/* Check infile pre-requisites */
|
||||
if (inFile==NULL) { DISPLAY("Pb opening %s \n", inFileName); return 11; }
|
||||
if (inFileSize==0) { DISPLAY("file is empty \n"); fclose(inFile); return 11; }
|
||||
if (benchedSize==0) { DISPLAY("not enough memory \n"); fclose(inFile); return 11; }
|
||||
|
||||
/* Memory size adjustments */
|
||||
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
|
||||
if (benchedSize < inFileSize) {
|
||||
DISPLAY("Not enough memory for '%s' full size; testing %i MB only... \n",
|
||||
inFileName, (int)(benchedSize>>20));
|
||||
}
|
||||
|
||||
/* Allocation */
|
||||
chunkP = (struct chunkParameters*) malloc(((benchedSize / (size_t)g_chunkSize)+1) * sizeof(struct chunkParameters));
|
||||
orig_buff = (char*) malloc(benchedSize);
|
||||
nbChunks = (int) ((benchedSize + (size_t)g_chunkSize - 1) / (size_t)g_chunkSize);
|
||||
maxCompressedChunkSize = LZ4_compressBound(g_chunkSize);
|
||||
compressedBuffSize = nbChunks * maxCompressedChunkSize;
|
||||
compressed_buff = (char*)malloc((size_t)compressedBuffSize);
|
||||
if(!chunkP || !orig_buff || !compressed_buff) {
|
||||
DISPLAY("\nError: not enough memory! \n");
|
||||
fclose(inFile);
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
return(12);
|
||||
}
|
||||
|
||||
/* Fill in src buffer */
|
||||
DISPLAY("Loading %s... \r", inFileName);
|
||||
readSize = fread(orig_buff, 1, benchedSize, inFile);
|
||||
fclose(inFile);
|
||||
|
||||
if (readSize != benchedSize) {
|
||||
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
return 13;
|
||||
}
|
||||
|
||||
/* Calculating input Checksum */
|
||||
crcOriginal = XXH32(orig_buff, benchedSize,0);
|
||||
|
||||
|
||||
/* Bench */
|
||||
{ int loopNb, nb_loops, chunkNb, cAlgNb, dAlgNb;
|
||||
size_t cSize=0;
|
||||
double ratio=0.;
|
||||
|
||||
DISPLAY("\r%79s\r", "");
|
||||
DISPLAY(" %s : \n", inFileName);
|
||||
|
||||
/* Bench Compression Algorithms */
|
||||
for (cAlgNb=0; (cAlgNb <= NB_COMPRESSION_ALGORITHMS) && (g_compressionTest); cAlgNb++) {
|
||||
const char* compressorName;
|
||||
int (*compressionFunction)(const char*, char*, int);
|
||||
void (*initFunction)(void) = NULL;
|
||||
double bestTime = 100000000.;
|
||||
|
||||
/* filter compressionAlgo only */
|
||||
if ((g_compressionAlgo != ALL_COMPRESSORS) && (g_compressionAlgo != cAlgNb)) continue;
|
||||
|
||||
/* Init data chunks */
|
||||
{ int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressed_buff;
|
||||
assert(nbChunks >= 1);
|
||||
for (i=0; i<nbChunks; i++) {
|
||||
chunkP[i].id = (U32)i;
|
||||
chunkP[i].origBuffer = in; in += g_chunkSize;
|
||||
assert(g_chunkSize > 0);
|
||||
if (remaining > (size_t)g_chunkSize) {
|
||||
chunkP[i].origSize = g_chunkSize;
|
||||
remaining -= (size_t)g_chunkSize;
|
||||
} else {
|
||||
chunkP[i].origSize = (int)remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
switch(cAlgNb)
|
||||
{
|
||||
case 0 : DISPLAY("Compression functions : \n"); continue;
|
||||
case 1 : compressionFunction = local_LZ4_compress_default_large; compressorName = "LZ4_compress_default"; break;
|
||||
case 2 : compressionFunction = local_LZ4_compress_default_small; compressorName = "LZ4_compress_default(small dst)"; break;
|
||||
case 3 : compressionFunction = local_LZ4_compress_destSize; compressorName = "LZ4_compress_destSize"; break;
|
||||
case 4 : compressionFunction = local_LZ4_compress_fast0; compressorName = "LZ4_compress_fast(0)"; break;
|
||||
case 5 : compressionFunction = local_LZ4_compress_fast1; compressorName = "LZ4_compress_fast(1)"; break;
|
||||
case 6 : compressionFunction = local_LZ4_compress_fast2; compressorName = "LZ4_compress_fast(2)"; break;
|
||||
case 7 : compressionFunction = local_LZ4_compress_fast17; compressorName = "LZ4_compress_fast(17)"; break;
|
||||
case 8 : compressionFunction = local_LZ4_compress_fast_extState0; compressorName = "LZ4_compress_fast_extState(0)"; break;
|
||||
case 9 : compressionFunction = local_LZ4_compress_fast_continue0; initFunction = local_LZ4_createStream; compressorName = "LZ4_compress_fast_continue(0)"; break;
|
||||
|
||||
case 10: compressionFunction = local_LZ4_compress_HC; compressorName = "LZ4_compress_HC"; break;
|
||||
case 12: compressionFunction = local_LZ4_compress_HC_extStateHC; compressorName = "LZ4_compress_HC_extStateHC"; break;
|
||||
case 14: compressionFunction = local_LZ4_compress_HC_continue; initFunction = local_LZ4_resetStreamHC; compressorName = "LZ4_compress_HC_continue"; break;
|
||||
#ifndef LZ4_DLL_IMPORT
|
||||
case 20: compressionFunction = local_LZ4_compress_forceDict; initFunction = local_LZ4_resetDictT; compressorName = "LZ4_compress_forceDict"; break;
|
||||
#endif
|
||||
case 30: compressionFunction = local_LZ4F_compressFrame; compressorName = "LZ4F_compressFrame";
|
||||
chunkP[0].origSize = (int)benchedSize; nbChunks=1;
|
||||
break;
|
||||
case 40: compressionFunction = local_LZ4_saveDict; compressorName = "LZ4_saveDict";
|
||||
if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; }
|
||||
LZ4_loadDict(&LZ4_stream, chunkP[0].origBuffer, chunkP[0].origSize);
|
||||
break;
|
||||
case 41: compressionFunction = local_LZ4_saveDictHC; compressorName = "LZ4_saveDictHC";
|
||||
if (chunkP[0].origSize < 8) { DISPLAY(" cannot bench %s with less then 8 bytes \n", compressorName); continue; }
|
||||
LZ4_loadDictHC(&LZ4_streamHC, chunkP[0].origBuffer, chunkP[0].origSize);
|
||||
break;
|
||||
default :
|
||||
continue; /* unknown ID : just skip */
|
||||
}
|
||||
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
double averageTime;
|
||||
clock_t clockTime;
|
||||
|
||||
PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, compressorName, (int)benchedSize);
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) compressed_buff[i]=(char)i; } /* warming up memory */
|
||||
|
||||
nb_loops = 0;
|
||||
clockTime = clock();
|
||||
while(clock() == clockTime);
|
||||
clockTime = clock();
|
||||
while(BMK_GetClockSpan(clockTime) < TIMELOOP) {
|
||||
if (initFunction!=NULL) initFunction();
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
|
||||
chunkP[chunkNb].compressedSize = compressionFunction(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) {
|
||||
DISPLAY("ERROR ! %s() = 0 !! \n", compressorName);
|
||||
exit(1);
|
||||
} }
|
||||
nb_loops++;
|
||||
}
|
||||
clockTime = BMK_GetClockSpan(clockTime);
|
||||
|
||||
nb_loops += !nb_loops; /* avoid division by zero */
|
||||
averageTime = ((double)clockTime) / nb_loops / CLOCKS_PER_SEC;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
cSize=0; for (chunkNb=0; chunkNb<nbChunks; chunkNb++) cSize += (size_t)chunkP[chunkNb].compressedSize;
|
||||
ratio = (double)cSize/(double)benchedSize*100.;
|
||||
PROGRESS("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\r", loopNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
|
||||
}
|
||||
|
||||
if (ratio<100.)
|
||||
DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.2f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 1000000);
|
||||
else
|
||||
DISPLAY("%2i-%-34.34s :%10i ->%9i (%5.1f%%),%7.1f MB/s\n", cAlgNb, compressorName, (int)benchedSize, (int)cSize, ratio, (double)benchedSize / bestTime / 100000);
|
||||
}
|
||||
|
||||
/* Prepare layout for decompression */
|
||||
/* Init data chunks */
|
||||
{ int i;
|
||||
size_t remaining = benchedSize;
|
||||
char* in = orig_buff;
|
||||
char* out = compressed_buff;
|
||||
|
||||
nbChunks = (int) (((int)benchedSize + (g_chunkSize-1))/ g_chunkSize);
|
||||
for (i=0; i<nbChunks; i++) {
|
||||
chunkP[i].id = (U32)i;
|
||||
chunkP[i].origBuffer = in; in += g_chunkSize;
|
||||
if ((int)remaining > g_chunkSize) {
|
||||
chunkP[i].origSize = g_chunkSize;
|
||||
remaining -= (size_t)g_chunkSize;
|
||||
} else {
|
||||
chunkP[i].origSize = (int)remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
chunkP[i].compressedBuffer = out; out += maxCompressedChunkSize;
|
||||
chunkP[i].compressedSize = 0;
|
||||
}
|
||||
}
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
|
||||
chunkP[chunkNb].compressedSize = LZ4_compress_default(chunkP[chunkNb].origBuffer, chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origSize, maxCompressedChunkSize);
|
||||
if (chunkP[chunkNb].compressedSize==0) {
|
||||
DISPLAY("ERROR ! %s() = 0 !! \n", "LZ4_compress");
|
||||
exit(1);
|
||||
} }
|
||||
|
||||
/* Decompression Algorithms */
|
||||
for (dAlgNb=0; (dAlgNb <= NB_DECOMPRESSION_ALGORITHMS) && g_decompressionTest; dAlgNb++) {
|
||||
const char* dName = NULL;
|
||||
int (*decompressionFunction)(const char*, char*, int, int) = NULL;
|
||||
double bestTime = 100000000.;
|
||||
int checkResult = 1;
|
||||
|
||||
if ((g_decompressionAlgo != ALL_DECOMPRESSORS) && (g_decompressionAlgo != dAlgNb)) continue;
|
||||
|
||||
switch(dAlgNb)
|
||||
{
|
||||
case 0: DISPLAY("Decompression functions : \n"); continue;
|
||||
case 1: decompressionFunction = local_LZ4_decompress_fast; dName = "LZ4_decompress_fast"; break;
|
||||
case 2: decompressionFunction = local_LZ4_decompress_fast_usingDict_prefix; dName = "LZ4_decompress_fast_usingDict(prefix)"; break;
|
||||
case 3: decompressionFunction = local_LZ4_decompress_fast_usingExtDict; dName = "LZ4_decompress_fast_using(Ext)Dict"; break;
|
||||
case 4: decompressionFunction = LZ4_decompress_safe; dName = "LZ4_decompress_safe"; break;
|
||||
case 5: decompressionFunction = local_LZ4_decompress_safe_withPrefix64k; dName = "LZ4_decompress_safe_withPrefix64k"; break;
|
||||
case 6: decompressionFunction = local_LZ4_decompress_safe_usingDict; dName = "LZ4_decompress_safe_usingDict"; break;
|
||||
case 7: decompressionFunction = local_LZ4_decompress_safe_partial; dName = "LZ4_decompress_safe_partial"; checkResult = 0; break;
|
||||
#ifndef LZ4_DLL_IMPORT
|
||||
case 8: decompressionFunction = local_LZ4_decompress_safe_forceExtDict; dName = "LZ4_decompress_safe_forceExtDict"; break;
|
||||
#endif
|
||||
case 10:
|
||||
case 11:
|
||||
case 12:
|
||||
if (dAlgNb == 10) { decompressionFunction = local_LZ4F_decompress; dName = "LZ4F_decompress"; } /* can be skipped */
|
||||
if (dAlgNb == 11) { decompressionFunction = local_LZ4F_decompress_followHint; dName = "LZ4F_decompress_followHint"; } /* can be skipped */
|
||||
if (dAlgNb == 12) { decompressionFunction = local_LZ4F_decompress_noHint; dName = "LZ4F_decompress_noHint"; } /* can be skipped */
|
||||
/* prepare compressed data using frame format */
|
||||
{ size_t const fcsize = LZ4F_compressFrame(compressed_buff, (size_t)compressedBuffSize, orig_buff, benchedSize, NULL);
|
||||
assert(!LZ4F_isError(fcsize));
|
||||
chunkP[0].origSize = (int)benchedSize;
|
||||
chunkP[0].compressedSize = (int)fcsize;
|
||||
nbChunks = 1;
|
||||
break;
|
||||
}
|
||||
default :
|
||||
continue; /* skip if unknown ID */
|
||||
}
|
||||
|
||||
assert(decompressionFunction != NULL);
|
||||
assert(dName != NULL);
|
||||
|
||||
{ size_t i; for (i=0; i<benchedSize; i++) orig_buff[i]=0; } /* zeroing source area, for CRC checking */
|
||||
|
||||
for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) {
|
||||
double averageTime;
|
||||
clock_t clockTime;
|
||||
U32 crcDecoded;
|
||||
|
||||
PROGRESS("%2i-%-34.34s :%10i ->\r", loopNb, dName, (int)benchedSize);
|
||||
|
||||
nb_loops = 0;
|
||||
clockTime = clock();
|
||||
while(clock() == clockTime);
|
||||
clockTime = clock();
|
||||
while(BMK_GetClockSpan(clockTime) < TIMELOOP) {
|
||||
for (chunkNb=0; chunkNb<nbChunks; chunkNb++) {
|
||||
int const decodedSize = decompressionFunction(chunkP[chunkNb].compressedBuffer, chunkP[chunkNb].origBuffer,
|
||||
chunkP[chunkNb].compressedSize, chunkP[chunkNb].origSize);
|
||||
if (chunkP[chunkNb].origSize != decodedSize) {
|
||||
DISPLAY("ERROR ! %s() == %i != %i !! \n",
|
||||
dName, decodedSize, chunkP[chunkNb].origSize);
|
||||
exit(1);
|
||||
} }
|
||||
nb_loops++;
|
||||
}
|
||||
clockTime = BMK_GetClockSpan(clockTime);
|
||||
|
||||
nb_loops += !nb_loops; /* Avoid division by zero */
|
||||
averageTime = (double)clockTime / nb_loops / CLOCKS_PER_SEC;
|
||||
if (averageTime < bestTime) bestTime = averageTime;
|
||||
|
||||
PROGRESS("%2i-%-34.34s :%10i -> %7.1f MB/s\r", loopNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
|
||||
|
||||
/* CRC Checking */
|
||||
crcDecoded = XXH32(orig_buff, benchedSize, 0);
|
||||
if (checkResult && (crcOriginal!=crcDecoded)) {
|
||||
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n",
|
||||
inFileName, (unsigned)crcOriginal, (unsigned)crcDecoded);
|
||||
exit(1);
|
||||
} }
|
||||
|
||||
DISPLAY("%2i-%-34.34s :%10i -> %7.1f MB/s\n", dAlgNb, dName, (int)benchedSize, (double)benchedSize / bestTime / 1000000);
|
||||
}
|
||||
}
|
||||
free(orig_buff);
|
||||
free(compressed_buff);
|
||||
free(chunkP);
|
||||
}
|
||||
|
||||
LZ4F_freeDecompressionContext(g_dCtx);
|
||||
if (g_pause) { printf("press enter...\n"); (void)getchar(); }
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static int usage(const char* exename)
|
||||
{
|
||||
DISPLAY( "Usage :\n");
|
||||
DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
|
||||
DISPLAY( "Arguments :\n");
|
||||
DISPLAY( " -c : compression tests only\n");
|
||||
DISPLAY( " -d : decompression tests only\n");
|
||||
DISPLAY( " -H/-h : Help (this text + advanced options)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int usage_advanced(void)
|
||||
{
|
||||
DISPLAY( "\nAdvanced options :\n");
|
||||
DISPLAY( " -c# : test only compression function # [1-%i]\n", NB_COMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -d# : test only decompression function # [1-%i]\n", NB_DECOMPRESSION_ALGORITHMS);
|
||||
DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
|
||||
DISPLAY( " -B# : Block size [4-7](default : 7)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int badusage(const char* exename)
|
||||
{
|
||||
DISPLAY("Wrong parameters\n");
|
||||
usage(exename);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
int i,
|
||||
filenamesStart=2;
|
||||
const char* exename = argv[0];
|
||||
const char* input_filename=0;
|
||||
|
||||
// Welcome message
|
||||
DISPLAY(WELCOME_MESSAGE);
|
||||
|
||||
if (argc<2) { badusage(exename); return 1; }
|
||||
|
||||
for(i=1; i<argc; i++) {
|
||||
const char* argument = argv[i];
|
||||
|
||||
if(!argument) continue; // Protection if argument empty
|
||||
if (!strcmp(argument, "--no-prompt")) {
|
||||
g_noPrompt = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Decode command (note : aggregated commands are allowed)
|
||||
if (argument[0]=='-') {
|
||||
while (argument[1]!=0) {
|
||||
argument ++;
|
||||
|
||||
switch(argument[0])
|
||||
{
|
||||
// Select compression algorithm only
|
||||
case 'c':
|
||||
g_decompressionTest = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
g_compressionAlgo *= 10;
|
||||
g_compressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Select decompression algorithm only
|
||||
case 'd':
|
||||
g_compressionTest = 0;
|
||||
while ((argument[1]>= '0') && (argument[1]<= '9')) {
|
||||
g_decompressionAlgo *= 10;
|
||||
g_decompressionAlgo += argument[1] - '0';
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Display help on usage
|
||||
case 'h' :
|
||||
case 'H': usage(exename); usage_advanced(); return 0;
|
||||
|
||||
// Modify Block Properties
|
||||
case 'B':
|
||||
while (argument[1]!=0)
|
||||
switch(argument[1])
|
||||
{
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
{ int B = argument[1] - '0';
|
||||
int S = 1 << (8 + 2*B);
|
||||
BMK_setBlocksize(S);
|
||||
argument++;
|
||||
break;
|
||||
}
|
||||
case 'D': argument++; break;
|
||||
default : goto _exit_blockProperties;
|
||||
}
|
||||
_exit_blockProperties:
|
||||
break;
|
||||
|
||||
// Modify Nb Iterations
|
||||
case 'i':
|
||||
if ((argument[1] >='0') && (argument[1] <='9')) {
|
||||
int iters = argument[1] - '0';
|
||||
BMK_setNbIterations(iters);
|
||||
argument++;
|
||||
}
|
||||
break;
|
||||
|
||||
// Pause at the end (hidden option)
|
||||
case 'p': BMK_setPause(); break;
|
||||
|
||||
// Unknown command
|
||||
default : badusage(exename); return 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// first provided filename is input
|
||||
if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
|
||||
|
||||
}
|
||||
|
||||
// No input filename ==> Error
|
||||
if(!input_filename) { badusage(exename); return 1; }
|
||||
|
||||
return fullSpeedBench(argv+filenamesStart, argc-filenamesStart);
|
||||
|
||||
}
|
||||
1841
lz4/tests/fuzzer.c
Normal file
1841
lz4/tests/fuzzer.c
Normal file
File diff suppressed because it is too large
Load diff
248
lz4/tests/roundTripTest.c
Normal file
248
lz4/tests/roundTripTest.c
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
/*
|
||||
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This program takes a file in input,
|
||||
* performs an LZ4 round-trip test (compress + decompress)
|
||||
* compares the result with original
|
||||
* and generates an abort() on corruption detection,
|
||||
* in order for afl to register the event as a crash.
|
||||
*/
|
||||
|
||||
|
||||
/*===========================================
|
||||
* Tuning Constant
|
||||
*==========================================*/
|
||||
#ifndef MIN_CLEVEL
|
||||
# define MIN_CLEVEL (int)(-5)
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/*===========================================
|
||||
* Dependencies
|
||||
*==========================================*/
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, free, exit */
|
||||
#include <stdio.h> /* fprintf */
|
||||
#include <string.h> /* strcmp */
|
||||
#include <assert.h>
|
||||
#include <sys/types.h> /* stat */
|
||||
#include <sys/stat.h> /* stat */
|
||||
#include "xxhash.h"
|
||||
|
||||
#include "lz4.h"
|
||||
#include "lz4hc.h"
|
||||
|
||||
|
||||
/*===========================================
|
||||
* Macros
|
||||
*==========================================*/
|
||||
#define MIN(a,b) ( (a) < (b) ? (a) : (b) )
|
||||
|
||||
#define MSG(...) fprintf(stderr, __VA_ARGS__)
|
||||
|
||||
#define CONTROL_MSG(c, ...) { \
|
||||
if ((c)) { \
|
||||
MSG(__VA_ARGS__); \
|
||||
MSG(" \n"); \
|
||||
abort(); \
|
||||
} \
|
||||
}
|
||||
|
||||
|
||||
static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
|
||||
{
|
||||
const char* const ip1 = (const char*)buff1;
|
||||
const char* const ip2 = (const char*)buff2;
|
||||
size_t pos;
|
||||
|
||||
for (pos=0; pos<buffSize; pos++)
|
||||
if (ip1[pos]!=ip2[pos])
|
||||
break;
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
|
||||
/* select a compression level
|
||||
* based on first bytes present in a reference buffer */
|
||||
static int select_clevel(const void* refBuff, size_t refBuffSize)
|
||||
{
|
||||
const int minCLevel = MIN_CLEVEL;
|
||||
const int maxClevel = LZ4HC_CLEVEL_MAX;
|
||||
const int cLevelSpan = maxClevel - minCLevel;
|
||||
size_t const hashLength = MIN(16, refBuffSize);
|
||||
unsigned const h32 = XXH32(refBuff, hashLength, 0);
|
||||
int const randL = h32 % (cLevelSpan+1);
|
||||
|
||||
return minCLevel + randL;
|
||||
}
|
||||
|
||||
|
||||
typedef int (*compressFn)(const char* src, char* dst, int srcSize, int dstSize, int cLevel);
|
||||
|
||||
|
||||
/** roundTripTest() :
|
||||
* Compresses `srcBuff` into `compressedBuff`,
|
||||
* then decompresses `compressedBuff` into `resultBuff`.
|
||||
* If clevel==0, compression level is derived from srcBuff's content head bytes.
|
||||
* This function abort() if it detects any round-trip error.
|
||||
* Therefore, if it returns, round trip is considered successfully validated.
|
||||
* Note : `compressedBuffCapacity` should be `>= LZ4_compressBound(srcSize)`
|
||||
* for compression to be guaranteed to work */
|
||||
static void roundTripTest(void* resultBuff, size_t resultBuffCapacity,
|
||||
void* compressedBuff, size_t compressedBuffCapacity,
|
||||
const void* srcBuff, size_t srcSize,
|
||||
int clevel)
|
||||
{
|
||||
int const proposed_clevel = clevel ? clevel : select_clevel(srcBuff, srcSize);
|
||||
int const selected_clevel = proposed_clevel < 0 ? -proposed_clevel : proposed_clevel; /* if level < 0, it becomes an accelearion value */
|
||||
compressFn compress = selected_clevel >= LZ4HC_CLEVEL_MIN ? LZ4_compress_HC : LZ4_compress_fast;
|
||||
int const cSize = compress((const char*)srcBuff, (char*)compressedBuff, (int)srcSize, (int)compressedBuffCapacity, selected_clevel);
|
||||
CONTROL_MSG(cSize == 0, "Compression error !");
|
||||
|
||||
{ int const dSize = LZ4_decompress_safe((const char*)compressedBuff, (char*)resultBuff, cSize, (int)resultBuffCapacity);
|
||||
CONTROL_MSG(dSize < 0, "Decompression detected an error !");
|
||||
CONTROL_MSG(dSize != (int)srcSize, "Decompression corruption error : wrong decompressed size !");
|
||||
}
|
||||
|
||||
/* check potential content corruption error */
|
||||
assert(resultBuffCapacity >= srcSize);
|
||||
{ size_t const errorPos = checkBuffers(srcBuff, resultBuff, srcSize);
|
||||
CONTROL_MSG(errorPos != srcSize,
|
||||
"Silent decoding corruption, at pos %u !!!",
|
||||
(unsigned)errorPos);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static void roundTripCheck(const void* srcBuff, size_t srcSize, int clevel)
|
||||
{
|
||||
size_t const cBuffSize = LZ4_compressBound((int)srcSize);
|
||||
void* const cBuff = malloc(cBuffSize);
|
||||
void* const rBuff = malloc(cBuffSize);
|
||||
|
||||
if (!cBuff || !rBuff) {
|
||||
fprintf(stderr, "not enough memory ! \n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
roundTripTest(rBuff, cBuffSize,
|
||||
cBuff, cBuffSize,
|
||||
srcBuff, srcSize,
|
||||
clevel);
|
||||
|
||||
free(rBuff);
|
||||
free(cBuff);
|
||||
}
|
||||
|
||||
|
||||
static size_t getFileSize(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
|
||||
#endif
|
||||
return (size_t)statbuf.st_size;
|
||||
}
|
||||
|
||||
|
||||
static int isDirectory(const char* infilename)
|
||||
{
|
||||
int r;
|
||||
#if defined(_MSC_VER)
|
||||
struct _stat64 statbuf;
|
||||
r = _stat64(infilename, &statbuf);
|
||||
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
|
||||
#else
|
||||
struct stat statbuf;
|
||||
r = stat(infilename, &statbuf);
|
||||
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/** loadFile() :
|
||||
* requirement : `buffer` size >= `fileSize` */
|
||||
static void loadFile(void* buffer, const char* fileName, size_t fileSize)
|
||||
{
|
||||
FILE* const f = fopen(fileName, "rb");
|
||||
if (isDirectory(fileName)) {
|
||||
MSG("Ignoring %s directory \n", fileName);
|
||||
exit(2);
|
||||
}
|
||||
if (f==NULL) {
|
||||
MSG("Impossible to open %s \n", fileName);
|
||||
exit(3);
|
||||
}
|
||||
{ size_t const readSize = fread(buffer, 1, fileSize, f);
|
||||
if (readSize != fileSize) {
|
||||
MSG("Error reading %s \n", fileName);
|
||||
exit(5);
|
||||
} }
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
|
||||
static void fileCheck(const char* fileName, int clevel)
|
||||
{
|
||||
size_t const fileSize = getFileSize(fileName);
|
||||
void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */);
|
||||
if (!buffer) {
|
||||
MSG("not enough memory \n");
|
||||
exit(4);
|
||||
}
|
||||
loadFile(buffer, fileName, fileSize);
|
||||
roundTripCheck(buffer, fileSize, clevel);
|
||||
free (buffer);
|
||||
}
|
||||
|
||||
|
||||
int bad_usage(const char* exeName)
|
||||
{
|
||||
MSG(" \n");
|
||||
MSG("bad usage: \n");
|
||||
MSG(" \n");
|
||||
MSG("%s [Options] fileName \n", exeName);
|
||||
MSG(" \n");
|
||||
MSG("Options: \n");
|
||||
MSG("-# : use #=[0-9] compression level (default:0 == random) \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
int main(int argCount, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
int argNb = 1;
|
||||
int clevel = 0;
|
||||
|
||||
assert(argCount >= 1);
|
||||
if (argCount < 2) return bad_usage(exeName);
|
||||
|
||||
if (argv[1][0] == '-') {
|
||||
clevel = argv[1][1] - '0';
|
||||
argNb = 2;
|
||||
}
|
||||
|
||||
if (argNb >= argCount) return bad_usage(exeName);
|
||||
|
||||
fileCheck(argv[argNb], clevel);
|
||||
MSG("no pb detected \n");
|
||||
return 0;
|
||||
}
|
||||
282
lz4/tests/test-lz4-list.py
Normal file
282
lz4/tests/test-lz4-list.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
#! /usr/bin/env python3
|
||||
import subprocess
|
||||
import time
|
||||
import glob
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SIZES = [3, 11] # Always 2 sizes
|
||||
MIB = 1048576
|
||||
LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../lz4"
|
||||
if not os.path.exists(LZ4):
|
||||
LZ4 = os.path.dirname(os.path.realpath(__file__)) + "/../programs/lz4"
|
||||
TEMP = tempfile.gettempdir()
|
||||
|
||||
|
||||
class NVerboseFileInfo(object):
|
||||
def __init__(self, line_in):
|
||||
self.line = line_in
|
||||
splitlines = line_in.split()
|
||||
if len(splitlines) != 7:
|
||||
errout("Unexpected line: {}".format(line_in))
|
||||
self.frames, self.type, self.block, self.compressed, self.uncompressed, self.ratio, self.filename = splitlines
|
||||
self.exp_unc_size = 0
|
||||
# Get real file sizes
|
||||
if "concat-all" in self.filename or "2f--content-size" in self.filename:
|
||||
for i in SIZES:
|
||||
self.exp_unc_size += os.path.getsize("{}/test_list_{}M".format(TEMP, i))
|
||||
else:
|
||||
uncompressed_filename = self.filename.split("-")[0]
|
||||
self.exp_unc_size += os.path.getsize("{}/{}".format(TEMP, uncompressed_filename))
|
||||
self.exp_comp_size = os.path.getsize("{}/{}".format(TEMP, self.filename))
|
||||
|
||||
|
||||
class TestNonVerbose(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(self):
|
||||
self.nvinfo_list = []
|
||||
for i, line in enumerate(execute("{} --list -m {}/test_list_*.lz4".format(LZ4, TEMP), print_output=True)):
|
||||
if i > 0:
|
||||
self.nvinfo_list.append(NVerboseFileInfo(line))
|
||||
|
||||
def test_frames(self):
|
||||
all_concat_frames = 0
|
||||
all_concat_index = None
|
||||
for i, nvinfo in enumerate(self.nvinfo_list):
|
||||
if "concat-all" in nvinfo.filename:
|
||||
all_concat_index = i
|
||||
elif "2f--content-size" in nvinfo.filename:
|
||||
self.assertEqual("2", nvinfo.frames, nvinfo.line)
|
||||
all_concat_frames += 2
|
||||
else:
|
||||
self.assertEqual("1", nvinfo.frames, nvinfo.line)
|
||||
all_concat_frames += 1
|
||||
self.assertNotEqual(None, all_concat_index, "Couldn't find concat-all file index.")
|
||||
self.assertEqual(self.nvinfo_list[all_concat_index].frames, str(all_concat_frames), self.nvinfo_list[all_concat_index].line)
|
||||
|
||||
def test_frame_types(self):
|
||||
for nvinfo in self.nvinfo_list:
|
||||
if "-lz4f-" in nvinfo.filename:
|
||||
self.assertEqual(nvinfo.type, "LZ4Frame", nvinfo.line)
|
||||
elif "-legc-" in nvinfo.filename:
|
||||
self.assertEqual(nvinfo.type, "LegacyFrame", nvinfo.line)
|
||||
elif "-skip-" in nvinfo.filename:
|
||||
self.assertEqual(nvinfo.type, "SkippableFrame", nvinfo.line)
|
||||
|
||||
def test_block(self):
|
||||
for nvinfo in self.nvinfo_list:
|
||||
# if "-leg" in nvinfo.filename or "-skip" in nvinfo.filename:
|
||||
# self.assertEqual(nvinfo.block, "-", nvinfo.line)
|
||||
if "--BD" in nvinfo.filename:
|
||||
self.assertRegex(nvinfo.block, "^B[0-9]+D$", nvinfo.line)
|
||||
elif "--BI" in nvinfo.filename:
|
||||
self.assertRegex(nvinfo.block, "^B[0-9]+I$", nvinfo.line)
|
||||
|
||||
def test_compressed_size(self):
|
||||
for nvinfo in self.nvinfo_list:
|
||||
self.assertEqual(nvinfo.compressed, to_human(nvinfo.exp_comp_size), nvinfo.line)
|
||||
|
||||
def test_ratio(self):
|
||||
for nvinfo in self.nvinfo_list:
|
||||
if "--content-size" in nvinfo.filename:
|
||||
self.assertEqual(nvinfo.ratio, "{:.2f}%".format(float(nvinfo.exp_comp_size) / float(nvinfo.exp_unc_size) * 100), nvinfo.line)
|
||||
|
||||
def test_uncompressed_size(self):
|
||||
for nvinfo in self.nvinfo_list:
|
||||
if "--content-size" in nvinfo.filename:
|
||||
self.assertEqual(nvinfo.uncompressed, to_human(nvinfo.exp_unc_size), nvinfo.line)
|
||||
|
||||
|
||||
class VerboseFileInfo(object):
|
||||
def __init__(self, lines):
|
||||
# Parse lines
|
||||
self.frame_list = []
|
||||
self.file_frame_map = []
|
||||
for i, line in enumerate(lines):
|
||||
if i == 0:
|
||||
self.filename = line
|
||||
continue
|
||||
elif i == 1:
|
||||
# Skip header
|
||||
continue
|
||||
frame_info = dict(zip(["frame", "type", "block", "checksum", "compressed", "uncompressed", "ratio"], line.split()))
|
||||
frame_info["line"] = line
|
||||
self.frame_list.append(frame_info)
|
||||
|
||||
|
||||
class TestVerbose(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(self):
|
||||
# Even do we're listing 2 files to test multiline working as expected.
|
||||
# we're only really interested in testing the output of the concat-all file.
|
||||
self.vinfo_list = []
|
||||
start = end = 0
|
||||
output = execute("{} --list -m -v {}/test_list_concat-all.lz4 {}/test_list_*M-lz4f-2f--content-size.lz4".format(LZ4, TEMP, TEMP), print_output=True)
|
||||
for i, line in enumerate(output):
|
||||
if line.startswith("test_list"):
|
||||
if start != 0 and end != 0:
|
||||
self.vinfo_list.append(VerboseFileInfo(output[start:end]))
|
||||
start = i
|
||||
if not line:
|
||||
end = i
|
||||
self.vinfo_list.append(VerboseFileInfo(output[start:end]))
|
||||
# Populate file_frame_map as a reference of the expected info
|
||||
concat_file_list = glob.glob("/tmp/test_list_[!concat]*.lz4")
|
||||
# One of the files has 2 frames so duplicate it in this list to map each frame 1 to a single file
|
||||
for i, filename in enumerate(concat_file_list):
|
||||
if "2f--content-size" in filename:
|
||||
concat_file_list.insert(i, filename)
|
||||
break
|
||||
self.cvinfo = self.vinfo_list[0]
|
||||
self.cvinfo.file_frame_map = concat_file_list
|
||||
self.cvinfo.compressed_size = os.path.getsize("{}/test_list_concat-all.lz4".format(TEMP))
|
||||
|
||||
def test_filename(self):
|
||||
for i, vinfo in enumerate(self.vinfo_list):
|
||||
self.assertRegex(vinfo.filename, "^test_list_.*({}/{})".format(i + 1, len(self.vinfo_list)))
|
||||
|
||||
def test_frame_number(self):
|
||||
for vinfo in self.vinfo_list:
|
||||
for i, frame_info in enumerate(vinfo.frame_list):
|
||||
self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"])
|
||||
|
||||
def test_frame_type(self):
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
if "-lz4f-" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"])
|
||||
elif "-legc-" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"])
|
||||
elif "-skip-" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"])
|
||||
|
||||
def test_block(self):
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
if "--BD" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertRegex(self.cvinfo.frame_list[i]["block"], "^B[0-9]+D$", self.cvinfo.frame_list[i]["line"])
|
||||
elif "--BI" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["block"], "^B[0-9]+I$", self.cvinfo.frame_list[i]["line"])
|
||||
|
||||
def test_checksum(self):
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"])
|
||||
|
||||
def test_compressed(self):
|
||||
total = 0
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
if "-2f-" not in self.cvinfo.file_frame_map[i]:
|
||||
expected_size = os.path.getsize(self.cvinfo.file_frame_map[i])
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"])
|
||||
total += int(self.cvinfo.frame_list[i]["compressed"])
|
||||
self.assertEqual(total, self.cvinfo.compressed_size, "Expected total sum ({}) to match {} filesize".format(total, self.cvinfo.filename))
|
||||
|
||||
def test_uncompressed(self):
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
ffm = self.cvinfo.file_frame_map[i]
|
||||
if "-2f-" not in ffm and "--content-size" in ffm:
|
||||
expected_size_unc = int(ffm[ffm.rindex("_") + 1:ffm.index("M")]) * 1048576
|
||||
self.assertEqual(self.cvinfo.frame_list[i]["uncompressed"], str(expected_size_unc), self.cvinfo.frame_list[i]["line"])
|
||||
|
||||
def test_ratio(self):
|
||||
for i, frame_info in enumerate(self.cvinfo.frame_list):
|
||||
if "--content-size" in self.cvinfo.file_frame_map[i]:
|
||||
self.assertEqual(self.cvinfo.frame_list[i]['ratio'],
|
||||
"{:.2f}%".format(float(self.cvinfo.frame_list[i]['compressed']) / float(self.cvinfo.frame_list[i]['uncompressed']) * 100),
|
||||
self.cvinfo.frame_list[i]["line"])
|
||||
|
||||
|
||||
def to_human(size):
|
||||
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']:
|
||||
if size < 1024.0:
|
||||
break
|
||||
size /= 1024.0
|
||||
return "{:.2f}{}".format(size, unit)
|
||||
|
||||
|
||||
def log(text):
|
||||
print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
|
||||
|
||||
|
||||
def errout(text, err=1):
|
||||
log(text)
|
||||
exit(err)
|
||||
|
||||
|
||||
def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
|
||||
if os.environ.get('QEMU_SYS'):
|
||||
command = "{} {}".format(os.environ['QEMU_SYS'], command)
|
||||
if print_command:
|
||||
log("> " + command)
|
||||
popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell)
|
||||
stdout_lines, stderr_lines = popen.communicate()
|
||||
stderr_lines = stderr_lines.decode("utf-8")
|
||||
stdout_lines = stdout_lines.decode("utf-8")
|
||||
if print_output:
|
||||
if stdout_lines:
|
||||
print(stdout_lines)
|
||||
if stderr_lines:
|
||||
print(stderr_lines)
|
||||
if popen.returncode is not None and popen.returncode != 0:
|
||||
if stderr_lines and not print_output and print_error:
|
||||
print(stderr_lines)
|
||||
errout("Failed to run: {}\n".format(command, stdout_lines + stderr_lines))
|
||||
return (stdout_lines + stderr_lines).splitlines()
|
||||
|
||||
|
||||
def cleanup(silent=False):
|
||||
for f in glob.glob("{}/test_list*".format(TEMP)):
|
||||
if not silent:
|
||||
log("Deleting {}".format(f))
|
||||
os.unlink(f)
|
||||
|
||||
|
||||
def datagen(file_name, size):
|
||||
non_sparse_size = size // 2
|
||||
sparse_size = size - non_sparse_size
|
||||
with open(file_name, "wb") as f:
|
||||
f.seek(sparse_size)
|
||||
f.write(os.urandom(non_sparse_size))
|
||||
|
||||
|
||||
def generate_files():
|
||||
# file format ~ test_list<frametype>-<no_frames>f<create-args>.lz4 ~
|
||||
# Generate LZ4Frames
|
||||
for i in SIZES:
|
||||
filename = "{}/test_list_{}M".format(TEMP, i)
|
||||
log("Generating {}".format(filename))
|
||||
datagen(filename, i * MIB)
|
||||
for j in ["--content-size", "-BI", "-BD", "-BX", "--no-frame-crc"]:
|
||||
lz4file = "{}-lz4f-1f{}.lz4".format(filename, j)
|
||||
execute("{} {} {} {}".format(LZ4, j, filename, lz4file))
|
||||
# Generate skippable frames
|
||||
lz4file = "{}-skip-1f.lz4".format(filename)
|
||||
skipsize = i * 1024
|
||||
skipbytes = bytes([80, 42, 77, 24]) + skipsize.to_bytes(4, byteorder='little', signed=False)
|
||||
with open(lz4file, 'wb') as f:
|
||||
f.write(skipbytes)
|
||||
f.write(os.urandom(skipsize))
|
||||
# Generate legacy frames
|
||||
lz4file = "{}-legc-1f.lz4".format(filename)
|
||||
execute("{} -l {} {}".format(LZ4, filename, lz4file))
|
||||
|
||||
# Concatenate --content-size files
|
||||
file_list = glob.glob("{}/test_list_*-lz4f-1f--content-size.lz4".format(TEMP))
|
||||
with open("{}/test_list_{}M-lz4f-2f--content-size.lz4".format(TEMP, sum(SIZES)), 'ab') as outfile:
|
||||
for fname in file_list:
|
||||
with open(fname, 'rb') as infile:
|
||||
outfile.write(infile.read())
|
||||
|
||||
# Concatenate all files
|
||||
file_list = glob.glob("{}/test_list_*.lz4".format(TEMP))
|
||||
with open("{}/test_list_concat-all.lz4".format(TEMP), 'ab') as outfile:
|
||||
for fname in file_list:
|
||||
with open(fname, 'rb') as infile:
|
||||
outfile.write(infile.read())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cleanup()
|
||||
generate_files()
|
||||
unittest.main(verbosity=2, exit=False)
|
||||
cleanup(silent=True)
|
||||
351
lz4/tests/test-lz4-speed.py
Normal file
351
lz4/tests/test-lz4-speed.py
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
#! /usr/bin/env python3
|
||||
|
||||
#
|
||||
# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
#
|
||||
|
||||
# Limitations:
|
||||
# - doesn't support filenames with spaces
|
||||
# - dir1/lz4 and dir2/lz4 will be merged in a single results file
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import string
|
||||
import subprocess
|
||||
import time
|
||||
import traceback
|
||||
import hashlib
|
||||
|
||||
script_version = 'v1.7.2 (2016-11-08)'
|
||||
default_repo_url = 'https://github.com/lz4/lz4.git'
|
||||
working_dir_name = 'speedTest'
|
||||
working_path = os.getcwd() + '/' + working_dir_name # /path/to/lz4/tests/speedTest
|
||||
clone_path = working_path + '/' + 'lz4' # /path/to/lz4/tests/speedTest/lz4
|
||||
email_header = 'lz4_speedTest'
|
||||
pid = str(os.getpid())
|
||||
verbose = False
|
||||
clang_version = "unknown"
|
||||
gcc_version = "unknown"
|
||||
args = None
|
||||
|
||||
|
||||
def hashfile(hasher, fname, blocksize=65536):
|
||||
with open(fname, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(blocksize), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def log(text):
|
||||
print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
|
||||
|
||||
|
||||
def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
|
||||
if print_command:
|
||||
log("> " + command)
|
||||
popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
|
||||
stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
|
||||
stderr_lines = stderr_lines.decode("utf-8")
|
||||
stdout_lines = stdout_lines.decode("utf-8")
|
||||
if print_output:
|
||||
if stdout_lines:
|
||||
print(stdout_lines)
|
||||
if stderr_lines:
|
||||
print(stderr_lines)
|
||||
if popen.returncode is not None and popen.returncode != 0:
|
||||
if stderr_lines and not print_output and print_error:
|
||||
print(stderr_lines)
|
||||
raise RuntimeError(stdout_lines + stderr_lines)
|
||||
return (stdout_lines + stderr_lines).splitlines()
|
||||
execute.cwd = None
|
||||
|
||||
|
||||
def does_command_exist(command):
|
||||
try:
|
||||
execute(command, verbose, False, False)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def send_email(emails, topic, text, have_mutt, have_mail):
|
||||
logFileName = working_path + '/' + 'tmpEmailContent'
|
||||
with open(logFileName, "w") as myfile:
|
||||
myfile.writelines(text)
|
||||
myfile.close()
|
||||
if have_mutt:
|
||||
execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
|
||||
elif have_mail:
|
||||
execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
|
||||
else:
|
||||
log("e-mail cannot be sent (mail or mutt not found)")
|
||||
|
||||
|
||||
def send_email_with_attachments(branch, commit, last_commit, args, text, results_files,
|
||||
logFileName, have_mutt, have_mail):
|
||||
with open(logFileName, "w") as myfile:
|
||||
myfile.writelines(text)
|
||||
myfile.close()
|
||||
email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
|
||||
% (email_header, pid, branch, commit, last_commit,
|
||||
args.lowerLimit, args.ratioLimit)
|
||||
if have_mutt:
|
||||
execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files
|
||||
+ ' < ' + logFileName)
|
||||
elif have_mail:
|
||||
execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName)
|
||||
else:
|
||||
log("e-mail cannot be sent (mail or mutt not found)")
|
||||
|
||||
|
||||
def git_get_branches():
|
||||
execute('git fetch -p', verbose)
|
||||
branches = execute('git branch -rl', verbose)
|
||||
output = []
|
||||
for line in branches:
|
||||
if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line):
|
||||
output.append(line.strip())
|
||||
return output
|
||||
|
||||
|
||||
def git_get_changes(branch, commit, last_commit):
|
||||
fmt = '--format="%h: (%an) %s, %ar"'
|
||||
if last_commit is None:
|
||||
commits = execute('git log -n 10 %s %s' % (fmt, commit))
|
||||
else:
|
||||
commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
|
||||
return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
|
||||
|
||||
|
||||
def get_last_results(resultsFileName):
|
||||
if not os.path.isfile(resultsFileName):
|
||||
return None, None, None, None
|
||||
commit = None
|
||||
csize = []
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
with open(resultsFileName, 'r') as f:
|
||||
for line in f:
|
||||
words = line.split()
|
||||
if len(words) <= 4: # branch + commit + compilerVer + md5
|
||||
commit = words[1]
|
||||
csize = []
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files"
|
||||
csize.append(int(words[1]))
|
||||
cspeed.append(float(words[3]))
|
||||
dspeed.append(float(words[5]))
|
||||
return commit, csize, cspeed, dspeed
|
||||
|
||||
|
||||
def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName,
|
||||
testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
|
||||
sleepTime = 30
|
||||
while os.getloadavg()[0] > args.maxLoadAvg:
|
||||
log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds"
|
||||
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
|
||||
time.sleep(sleepTime)
|
||||
start_load = str(os.getloadavg())
|
||||
result = execute('programs/%s -rqi5b1e%s %s' % (executableName, args.lastCLevel, testFilePath), print_output=True)
|
||||
end_load = str(os.getloadavg())
|
||||
linesExpected = args.lastCLevel + 1
|
||||
if len(result) != linesExpected:
|
||||
raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
|
||||
with open(resultsFileName, "a") as myfile:
|
||||
myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum))
|
||||
myfile.write('\n'.join(result) + '\n')
|
||||
myfile.close()
|
||||
if (last_cspeed == None):
|
||||
log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
|
||||
return ""
|
||||
commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
|
||||
text = ""
|
||||
for i in range(0, min(len(cspeed), len(last_cspeed))):
|
||||
print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName))
|
||||
if (cspeed[i]/last_cspeed[i] < args.lowerLimit):
|
||||
text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
|
||||
if (dspeed[i]/last_dspeed[i] < args.lowerLimit):
|
||||
text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
|
||||
if (float(last_csize[i])/csize[i] < args.ratioLimit):
|
||||
text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
|
||||
if text:
|
||||
text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text
|
||||
return text
|
||||
|
||||
|
||||
def update_config_file(branch, commit):
|
||||
last_commit = None
|
||||
commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
|
||||
if os.path.isfile(commitFileName):
|
||||
with open(commitFileName, 'r') as infile:
|
||||
last_commit = infile.read()
|
||||
with open(commitFileName, 'w') as outfile:
|
||||
outfile.write(commit)
|
||||
return last_commit
|
||||
|
||||
|
||||
def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName):
|
||||
last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
|
||||
if not args.dry_run:
|
||||
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
|
||||
if text:
|
||||
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
|
||||
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
|
||||
return text
|
||||
|
||||
|
||||
def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
|
||||
local_branch = branch.split('/')[1]
|
||||
version = local_branch.rpartition('-')[2] + '_' + commit
|
||||
if not args.dry_run:
|
||||
execute('make -C programs clean lz4 CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DLZ4_GIT_COMMIT=%s" && ' % version +
|
||||
'mv programs/lz4 programs/lz4_clang && ' +
|
||||
'make -C programs clean lz4 lz4c32 MOREFLAGS="-DLZ4_GIT_COMMIT=%s"' % version)
|
||||
md5_lz4 = hashfile(hashlib.md5(), clone_path + '/programs/lz4')
|
||||
md5_lz4c32 = hashfile(hashlib.md5(), clone_path + '/programs/lz4c32')
|
||||
md5_lz4_clang = hashfile(hashlib.md5(), clone_path + '/programs/lz4_clang')
|
||||
print("md5(lz4)=%s\nmd5(lz4c32)=%s\nmd5(lz4_clang)=%s" % (md5_lz4, md5_lz4c32, md5_lz4_clang))
|
||||
print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version))
|
||||
|
||||
logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
|
||||
text_to_send = []
|
||||
results_files = ""
|
||||
|
||||
for filePath in testFilePaths:
|
||||
fileName = filePath.rpartition('/')[2]
|
||||
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'lz4', md5_lz4, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'lz4c32', md5_lz4c32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'lz4_clang', md5_lz4_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
if text_to_send:
|
||||
send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('testFileNames', help='file or directory names list for speed benchmark')
|
||||
parser.add_argument('emails', help='list of e-mail addresses to send warnings')
|
||||
parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="")
|
||||
parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
|
||||
parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98)
|
||||
parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999)
|
||||
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
|
||||
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
|
||||
parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
|
||||
parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
|
||||
parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
|
||||
parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
|
||||
args = parser.parse_args()
|
||||
verbose = args.verbose
|
||||
|
||||
# check if test files are accessible
|
||||
testFileNames = args.testFileNames.split()
|
||||
testFilePaths = []
|
||||
for fileName in testFileNames:
|
||||
fileName = os.path.expanduser(fileName)
|
||||
if os.path.isfile(fileName) or os.path.isdir(fileName):
|
||||
testFilePaths.append(os.path.abspath(fileName))
|
||||
else:
|
||||
log("ERROR: File/directory not found: " + fileName)
|
||||
exit(1)
|
||||
|
||||
# check availability of e-mail senders
|
||||
have_mutt = does_command_exist("mutt -h")
|
||||
have_mail = does_command_exist("mail -V")
|
||||
if not have_mutt and not have_mail:
|
||||
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
|
||||
exit(1)
|
||||
|
||||
clang_version = execute("clang -v 2>&1 | grep 'clang version' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
|
||||
gcc_version = execute("gcc -dumpversion", verbose)[0];
|
||||
|
||||
if verbose:
|
||||
print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
|
||||
print("working_path=%s" % working_path)
|
||||
print("clone_path=%s" % clone_path)
|
||||
print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
|
||||
print("message=%s" % args.message)
|
||||
print("emails=%s" % args.emails)
|
||||
print("maxLoadAvg=%s" % args.maxLoadAvg)
|
||||
print("lowerLimit=%s" % args.lowerLimit)
|
||||
print("ratioLimit=%s" % args.ratioLimit)
|
||||
print("lastCLevel=%s" % args.lastCLevel)
|
||||
print("sleepTime=%s" % args.sleepTime)
|
||||
print("timeout=%s" % args.timeout)
|
||||
print("dry_run=%s" % args.dry_run)
|
||||
print("verbose=%s" % args.verbose)
|
||||
print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
|
||||
|
||||
# clone lz4 repo if needed
|
||||
if not os.path.isdir(working_path):
|
||||
os.mkdir(working_path)
|
||||
if not os.path.isdir(clone_path):
|
||||
execute.cwd = working_path
|
||||
execute('git clone ' + args.repoURL)
|
||||
if not os.path.isdir(clone_path):
|
||||
log("ERROR: lz4 clone not found: " + clone_path)
|
||||
exit(1)
|
||||
execute.cwd = clone_path
|
||||
|
||||
# check if speedTest.pid already exists
|
||||
pidfile = "./speedTest.pid"
|
||||
if os.path.isfile(pidfile):
|
||||
log("ERROR: %s already exists, exiting" % pidfile)
|
||||
exit(1)
|
||||
|
||||
send_email(args.emails, '[%s:%s] test-lz4-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
|
||||
with open(pidfile, 'w') as the_file:
|
||||
the_file.write(pid)
|
||||
|
||||
branch = ""
|
||||
commit = ""
|
||||
first_time = True
|
||||
while True:
|
||||
try:
|
||||
if first_time:
|
||||
first_time = False
|
||||
else:
|
||||
if verbose:
|
||||
log("sleep for %s seconds" % args.sleepTime)
|
||||
time.sleep(args.sleepTime)
|
||||
loadavg = os.getloadavg()[0]
|
||||
if (loadavg <= args.maxLoadAvg):
|
||||
branches = git_get_branches()
|
||||
for branch in branches:
|
||||
commit = execute('git show -s --format=%h ' + branch, verbose)[0]
|
||||
last_commit = update_config_file(branch, commit)
|
||||
if commit == last_commit:
|
||||
log("skipping branch %s: head %s already processed" % (branch, commit))
|
||||
else:
|
||||
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
|
||||
execute('git checkout -- . && git checkout ' + branch)
|
||||
print(git_get_changes(branch, commit, last_commit))
|
||||
test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
|
||||
else:
|
||||
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
|
||||
except Exception as e:
|
||||
stack = traceback.format_exc()
|
||||
email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
|
||||
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
|
||||
print(stack)
|
||||
except KeyboardInterrupt:
|
||||
os.unlink(pidfile)
|
||||
send_email(args.emails, '[%s:%s] test-lz4-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
|
||||
exit(0)
|
||||
156
lz4/tests/test-lz4-versions.py
Normal file
156
lz4/tests/test-lz4-versions.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Test LZ4 interoperability between versions"""
|
||||
|
||||
#
|
||||
# Copyright (C) 2011-present, Takayuki Matsuoka
|
||||
# All rights reserved.
|
||||
# GPL v2 License
|
||||
#
|
||||
|
||||
import glob
|
||||
import subprocess
|
||||
import filecmp
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import hashlib
|
||||
|
||||
repo_url = 'https://github.com/lz4/lz4.git'
|
||||
tmp_dir_name = 'tests/versionsTest'
|
||||
make_cmd = 'make'
|
||||
git_cmd = 'git'
|
||||
test_dat_src = 'README.md'
|
||||
test_dat = 'test_dat'
|
||||
head = 'v999'
|
||||
|
||||
def proc(cmd_args, pipe=True, dummy=False):
|
||||
if dummy:
|
||||
return
|
||||
if pipe:
|
||||
subproc = subprocess.Popen(cmd_args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
else:
|
||||
subproc = subprocess.Popen(cmd_args)
|
||||
return subproc.communicate()
|
||||
|
||||
def make(args, pipe=True):
|
||||
return proc([make_cmd] + args, pipe)
|
||||
|
||||
def git(args, pipe=True):
|
||||
return proc([git_cmd] + args, pipe)
|
||||
|
||||
def get_git_tags():
|
||||
stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
|
||||
tags = stdout.decode('utf-8').split()
|
||||
stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
|
||||
tags += stdout.decode('utf-8').split()
|
||||
return tags
|
||||
|
||||
# https://stackoverflow.com/a/19711609/2132223
|
||||
def sha1_of_file(filepath):
|
||||
with open(filepath, 'rb') as f:
|
||||
return hashlib.sha1(f.read()).hexdigest()
|
||||
|
||||
if __name__ == '__main__':
|
||||
error_code = 0
|
||||
base_dir = os.getcwd() + '/..' # /path/to/lz4
|
||||
tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/lz4/tests/versionsTest
|
||||
clone_dir = tmp_dir + '/' + 'lz4' # /path/to/lz4/tests/versionsTest/lz4
|
||||
programs_dir = base_dir + '/programs' # /path/to/lz4/programs
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
|
||||
# since Travis clones limited depth, we should clone full repository
|
||||
if not os.path.isdir(clone_dir):
|
||||
git(['clone', repo_url, clone_dir])
|
||||
|
||||
shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
|
||||
|
||||
# Retrieve all release tags
|
||||
print('Retrieve all release tags :')
|
||||
os.chdir(clone_dir)
|
||||
tags = [head] + get_git_tags()
|
||||
print(tags);
|
||||
|
||||
# Build all release lz4c and lz4c32
|
||||
for tag in tags:
|
||||
os.chdir(base_dir)
|
||||
dst_lz4c = '{}/lz4c.{}' .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG>
|
||||
dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG>
|
||||
if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head:
|
||||
if tag != head:
|
||||
r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/<TAG>
|
||||
os.makedirs(r_dir, exist_ok=True)
|
||||
os.chdir(clone_dir)
|
||||
git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
|
||||
os.chdir(r_dir + '/programs') # /path/to/lz4/lz4test/<TAG>/programs
|
||||
else:
|
||||
os.chdir(programs_dir)
|
||||
make(['clean', 'lz4c'], False)
|
||||
shutil.copy2('lz4c', dst_lz4c)
|
||||
make(['clean', 'lz4c32'], False)
|
||||
shutil.copy2('lz4c32', dst_lz4c32)
|
||||
|
||||
# Compress test.dat by all released lz4c and lz4c32
|
||||
print('Compress test.dat by all released lz4c and lz4c32')
|
||||
os.chdir(tmp_dir)
|
||||
for lz4 in glob.glob("*.lz4"):
|
||||
os.remove(lz4)
|
||||
for tag in tags:
|
||||
proc(['./lz4c.' + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4'])
|
||||
proc(['./lz4c.' + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4'])
|
||||
proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4'])
|
||||
proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4'])
|
||||
|
||||
print('Full list of compressed files')
|
||||
lz4s = sorted(glob.glob('*.lz4'))
|
||||
for lz4 in lz4s:
|
||||
print(lz4 + ' : ' + repr(os.path.getsize(lz4)))
|
||||
|
||||
# Remove duplicated .lz4 files
|
||||
print('')
|
||||
print('Duplicated files')
|
||||
lz4s = sorted(glob.glob('*.lz4'))
|
||||
for i, lz4 in enumerate(lz4s):
|
||||
if not os.path.isfile(lz4):
|
||||
continue
|
||||
for j in range(i+1, len(lz4s)):
|
||||
lz4t = lz4s[j]
|
||||
if not os.path.isfile(lz4t):
|
||||
continue
|
||||
if filecmp.cmp(lz4, lz4t):
|
||||
os.remove(lz4t)
|
||||
print('{} == {}'.format(lz4, lz4t))
|
||||
|
||||
print('Enumerate only different compressed files')
|
||||
lz4s = sorted(glob.glob('*.lz4'))
|
||||
for lz4 in lz4s:
|
||||
print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4))
|
||||
|
||||
# Decompress remained .lz4 files by all released lz4c and lz4c32
|
||||
print('Decompression tests and verifications')
|
||||
lz4s = sorted(glob.glob('*.lz4'))
|
||||
for dec in glob.glob("*.dec"):
|
||||
os.remove(dec)
|
||||
for lz4 in lz4s:
|
||||
print(lz4, end=" ")
|
||||
for tag in tags:
|
||||
print(tag, end=" ")
|
||||
proc(['./lz4c.' + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec'])
|
||||
proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec'])
|
||||
print(' OK') # well, here, decompression has worked; but file is not yet verified
|
||||
|
||||
# Compare all '.dec' files with test_dat
|
||||
decs = glob.glob('*.dec')
|
||||
for dec in decs:
|
||||
if not filecmp.cmp(dec, test_dat):
|
||||
print('ERR : ' + dec)
|
||||
error_code = 1
|
||||
else:
|
||||
print('OK : ' + dec)
|
||||
os.remove(dec)
|
||||
|
||||
if error_code != 0:
|
||||
print('ERROR')
|
||||
|
||||
sys.exit(error_code)
|
||||
72
lz4/tests/test_custom_block_sizes.sh
Executable file
72
lz4/tests/test_custom_block_sizes.sh
Executable file
|
|
@ -0,0 +1,72 @@
|
|||
#/usr/bin/env sh
|
||||
set -e
|
||||
|
||||
LZ4=../lz4
|
||||
CHECKFRAME=./checkFrame
|
||||
DATAGEN=./datagen
|
||||
|
||||
failures=""
|
||||
|
||||
TMPFILE=/tmp/test_custom_block_sizes.$$
|
||||
TMPFILE1=/tmp/test_custom_block_sizes1.$$
|
||||
TMPFILE2=/tmp/test_custom_block_sizes2.$$
|
||||
$DATAGEN -g12345678 > $TMPFILE1
|
||||
$DATAGEN -g12345678 > $TMPFILE2
|
||||
|
||||
echo Testing -B31
|
||||
$LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) "
|
||||
|
||||
for blocksize in 32 65535 65536
|
||||
do
|
||||
echo Testing -B$blocksize
|
||||
$LZ4 -f -B$blocksize $TMPFILE1
|
||||
$LZ4 -f -B$blocksize $TMPFILE2
|
||||
cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4
|
||||
$CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize "
|
||||
done
|
||||
|
||||
for blocksize in 65537 262143 262144
|
||||
do
|
||||
echo Testing -B$blocksize
|
||||
$LZ4 -f -B$blocksize $TMPFILE1
|
||||
$LZ4 -f -B$blocksize $TMPFILE2
|
||||
cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4
|
||||
$CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize "
|
||||
done
|
||||
|
||||
for blocksize in 262145 1048575 1048576
|
||||
do
|
||||
echo Testing -B$blocksize
|
||||
$LZ4 -f -B$blocksize $TMPFILE1
|
||||
$LZ4 -f -B$blocksize $TMPFILE2
|
||||
cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4
|
||||
$CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize "
|
||||
done
|
||||
|
||||
for blocksize in 1048577 4194303 4194304
|
||||
do
|
||||
echo Testing -B$blocksize
|
||||
$LZ4 -f -B$blocksize $TMPFILE1
|
||||
$LZ4 -f -B$blocksize $TMPFILE2
|
||||
cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4
|
||||
$CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize "
|
||||
done
|
||||
|
||||
for blocksize in 4194305 10485760
|
||||
do
|
||||
echo Testing -B$blocksize
|
||||
$LZ4 -f -B$blocksize $TMPFILE1
|
||||
$LZ4 -f -B$blocksize $TMPFILE2
|
||||
cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4
|
||||
$CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize "
|
||||
done
|
||||
|
||||
rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4
|
||||
if [ "$failures" == "" ]
|
||||
then
|
||||
echo ---- All tests passed
|
||||
exit 0
|
||||
else
|
||||
echo ---- The following tests had failures: $failures
|
||||
exit 1
|
||||
fi
|
||||
28
lz4/tests/test_install.sh
Executable file
28
lz4/tests/test_install.sh
Executable file
|
|
@ -0,0 +1,28 @@
|
|||
#/usr/bin/env sh
|
||||
set -e
|
||||
|
||||
|
||||
make="make -C $lz4_root"
|
||||
unamestr=$(uname)
|
||||
if [ "$unamestr" = 'Linux' ]; then
|
||||
make="make -C $lz4_root"
|
||||
elif [ "$unamestr" = 'FreeBSD' -o "$unamestr" = 'OpenBSD' ]; then
|
||||
make="gmake -C $lz4_root"
|
||||
fi
|
||||
|
||||
for cmd in install uninstall; do
|
||||
for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do
|
||||
lower=$(echo $upper | tr '[:upper:]' '[:lower:]')
|
||||
tmp_lower="$(pwd)/tmp-lower-$lower/"
|
||||
tmp_upper="$(pwd)/tmp-upper-$lower/"
|
||||
echo $make $cmd DESTDIR="$tmp_upper" $upper="test"
|
||||
$make $cmd DESTDIR="$tmp_upper" $upper="test" >/dev/null
|
||||
echo $make $cmd DESTDIR="$tmp_lower" $lower="test"
|
||||
$make $cmd DESTDIR="$tmp_lower" $lower="test" >/dev/null
|
||||
command diff -r "$tmp_lower" "$tmp_upper" && echo "SAME!" || false
|
||||
if [ "x$cmd" = "xuninstall" ]; then
|
||||
test -z "$(find "$tmp_lower" -type f)" && echo "EMPTY!" || false
|
||||
rm -rf "$tmp_upper" "$tmp_lower"
|
||||
fi
|
||||
done
|
||||
done
|
||||
Reference in a new issue