Emulation/RetroArch
1
0
Fork 0
mirror of https://github.com/libretro/RetroArch.git synced 2024-05-20 13:27:22 -04:00
Code Issues Packages Projects Releases Wiki Activity

[ORBIS] Update xxHash dependecy

Browse source
This commit is contained in:
Francisco Javier Trujillo Mata 2022-05-14 18:23:32 +02:00
parent f56f28d226
commit ba47d09c4b
45 changed files with 2358 additions and 1347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
deps/xxHash/CHANGELOG vendored
View file

@ -1,3 +1,22 @@
v0.8.1
- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc
- perf : improved XXH64 speed and latency on small inputs
- perf : small XXH32 speed and latency improvement on small inputs of random size
- perf : minor stack usage improvement for XXH32 and XXH64
- api : new experimental variants XXH3_*_withSecretandSeed()
- api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN)
- cli : xxhsum can now generate and check XXH3 checksums, using command `-H3`
- build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3
- build: fix xxh_x86dispatch build with MSVC, by @apankrat
- build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL
- build: improved PPC64LE vector support, by @mpe
- install: fix pkgconfig, by @ellert
- install: compatibility with Haiku, by @Begasus
- doc : code comments made compatible with doxygen, by @easyaspi314
- misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0
- misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat
- misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314
v0.8.0
- api : stabilize XXH3
- cli : xxhsum can parse BSD-style --check lines, by @WayneD

5
deps/xxHash/Doxyfile vendored
View file

@ -2,7 +2,7 @@
DOXYFILE_ENCODING = UTF-8
PROJECT_NAME = "xxHash"
PROJECT_NUMBER = "0.8.0"
PROJECT_NUMBER = "0.8.1"
PROJECT_BRIEF = "Extremely fast non-cryptographic hash function"
OUTPUT_DIRECTORY = doxygen
OUTPUT_LANGUAGE = English
@ -42,6 +42,9 @@ PREDEFINED = "XXH_DOXYGEN=" \
"XSUM_API=" \
"XXH_STATIC_LINKING_ONLY" \
"XXH_IMPLEMENTATION" \
"XXH_PUREF=[[gnu::pure]]" \
"XXH_CONSTF=[[gnu::const]]" \
"XXH_MALLOCF=[[gnu::malloc]]" \
"XXH_ALIGN(N)=alignas(N)" \
"XXH_ALIGN_MEMBER(align,type)=alignas(align) type"

24
deps/xxHash/LICENSE vendored
View file

@ -1,5 +1,5 @@
xxHash Library
Copyright (c) 2012-2020 Yann Collet
Copyright (c) 2012-2021 Yann Collet
All rights reserved.
BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
@ -24,25 +24,3 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----------------------------------------------------
xxhsum command line interface
Copyright (c) 2013-2020 Yann Collet
All rights reserved.
GPL v2 License
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

88
deps/xxHash/Makefile vendored
View file

@ -1,6 +1,6 @@
# ################################################################
# xxHash Makefile
# Copyright (C) 2012-2020 Yann Collet
# Copyright (C) 2012-2021 Yann Collet
#
# GPL v2 License
#
@ -75,13 +75,15 @@ XXHSUM_SRC_DIR = cli
XXHSUM_SPLIT_SRCS = $(XXHSUM_SRC_DIR)/xxhsum.c \
$(XXHSUM_SRC_DIR)/xsum_os_specific.c \
$(XXHSUM_SRC_DIR)/xsum_output.c \
$(XXHSUM_SRC_DIR)/xsum_sanity_check.c
$(XXHSUM_SRC_DIR)/xsum_sanity_check.c \
$(XXHSUM_SRC_DIR)/xsum_bench.c
XXHSUM_SPLIT_OBJS = $(XXHSUM_SPLIT_SRCS:.c=.o)
XXHSUM_HEADERS = $(XXHSUM_SRC_DIR)/xsum_config.h \
$(XXHSUM_SRC_DIR)/xsum_arch.h \
$(XXHSUM_SRC_DIR)/xsum_os_specific.h \
$(XXHSUM_SRC_DIR)/xsum_output.h \
$(XXHSUM_SRC_DIR)/xsum_sanity_check.h
$(XXHSUM_SRC_DIR)/xsum_sanity_check.h \
$(XXHSUM_SRC_DIR)/xsum_bench.h
## generate CLI and libraries in release mode (default for `make`)
.PHONY: default
@ -204,6 +206,8 @@ check: xxhsum ## basic tests for xxhsum CLI, set RUN_ENV for emulated environm
$(RUN_ENV) ./xxhsum$(EXT) -H0 xxhash.c
# 128-bit
$(RUN_ENV) ./xxhsum$(EXT) -H2 xxhash.c
# XXH3 (enforce BSD style)
$(RUN_ENV) ./xxhsum$(EXT) -H3 xxhash.c | grep "XXH3"
# request incorrect variant
$(RUN_ENV) ./xxhsum$(EXT) -H9 xxhash.c ; test $$? -eq 1
@printf "\n ....... checks completed successfully ....... \n"
@ -250,12 +254,18 @@ test-xxhsum-c: xxhsum
./xxhsum -H2 --tag $(TEST_FILES) > .test.xxh128_tag
./xxhsum -H2 --little-endian $(TEST_FILES) > .test.le_xxh128
./xxhsum -H2 --tag --little-endian $(TEST_FILES) > .test.le_xxh128_tag
./xxhsum -H3 $(TEST_FILES) > .test.xxh3
./xxhsum -H3 --tag $(TEST_FILES) > .test.xxh3_tag
./xxhsum -H3 --little-endian $(TEST_FILES) > .test.le_xxh3
./xxhsum -H3 --tag --little-endian $(TEST_FILES) > .test.le_xxh3_tag
./xxhsum -c .test.xxh*
./xxhsum -c --little-endian .test.le_xxh*
./xxhsum -c .test.*_tag
# read list of files from stdin
./xxhsum -c < .test.xxh64
./xxhsum -c < .test.xxh32
./xxhsum -c < .test.xxh64
./xxhsum -c < .test.xxh128
./xxhsum -c < .test.xxh3
cat .test.xxh* | ./xxhsum -c -
# check variant with '*' marker as second separator
$(SED) 's/ / \*/' .test.xxh32 | ./xxhsum -c
@ -264,12 +274,15 @@ test-xxhsum-c: xxhsum
./xxhsum --tag -H0 xxhsum* | $(GREP) XXH32
./xxhsum --tag -H1 xxhsum* | $(GREP) XXH64
./xxhsum --tag -H2 xxhsum* | $(GREP) XXH128
./xxhsum --tag -H3 xxhsum* | $(GREP) XXH3
./xxhsum -H3 xxhsum* | $(GREP) XXH3 # --tag is implicit for H3
./xxhsum --tag -H32 xxhsum* | $(GREP) XXH32
./xxhsum --tag -H64 xxhsum* | $(GREP) XXH64
./xxhsum --tag -H128 xxhsum* | $(GREP) XXH128
./xxhsum --tag -H0 --little-endian xxhsum* | $(GREP) XXH32_LE
./xxhsum --tag -H1 --little-endian xxhsum* | $(GREP) XXH64_LE
./xxhsum --tag -H2 --little-endian xxhsum* | $(GREP) XXH128_LE
./xxhsum -H3 --little-endian xxhsum* | $(GREP) XXH3_LE
./xxhsum --tag -H32 --little-endian xxhsum* | $(GREP) XXH32_LE
./xxhsum --tag -H64 --little-endian xxhsum* | $(GREP) XXH64_LE
./xxhsum --tag -H128 --little-endian xxhsum* | $(GREP) XXH128_LE
@ -289,6 +302,10 @@ test-xxhsum-c: xxhsum
echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
@$(RM) .test.*
.PHONY: test-filename-escape
test-filename-escape:
$(MAKE) -C tests test_filename_escape
.PHONY: armtest
armtest: clean
@echo ---- test ARM compilation ----
@ -319,14 +336,35 @@ c90test: xxhash.c
$(RM) xxhash.o
endif
.PHONY: noxxh3test
noxxh3test: CPPFLAGS += -DXXH_NO_XXH3
noxxh3test: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
noxxh3test: OFILE = xxh_noxxh3.o
noxxh3test: xxhash.c
@echo ---- test compilation without XXH3 ----
$(RM) xxhash.o
$(CC) $(FLAGS) $^ -c
$(NM) xxhash.o | $(GREP) XXH3_ ; test $$? -eq 1
$(RM) xxhash.o
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) XXH3_ ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: nostreamtest
nostreamtest: CPPFLAGS += -DXXH_NO_STREAM
nostreamtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
nostreamtest: OFILE = xxh_nostream.o
nostreamtest: xxhash.c
@echo ---- test compilation without streaming ----
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) update ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: nostdlibtest
nostdlibtest: CPPFLAGS += -DXXH_NO_STDLIB
nostdlibtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
nostdlibtest: OFILE = xxh_nostdlib.o
nostdlibtest: xxhash.c
@echo ---- test compilation without \<stdlib.h\> ----
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) "U _free\|U free" ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: usan
usan: CC=clang
@ -374,7 +412,7 @@ preview-man: man
.PHONY: test
test: DEBUGFLAGS += -DXXH_DEBUGLEVEL=1
test: all namespaceTest check test-xxhsum-c c90test test-tools
test: all namespaceTest check test-xxhsum-c c90test test-tools noxxh3test nostdlibtest
.PHONY: test-inline
test-inline:
@ -444,6 +482,7 @@ endif
INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_DATA ?= $(INSTALL) -m 644
INSTALL_DIR ?= $(INSTALL) -d -m 755
# Escape special symbols by putting each character into its separate class
@ -480,11 +519,14 @@ libxxhash.pc: libxxhash.pc.in
$< > $@
.PHONY: install
install: lib libxxhash.pc xxhsum ## install libraries, CLI, links and man page
@echo Installing libxxhash
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)
install_libxxhash.a: libxxhash.a
@echo Installing libxxhash.a
$(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_DATA) libxxhash.a $(DESTDIR)$(LIBDIR)
install_libxxhash: libxxhash
@echo Installing libxxhash
$(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_PROGRAM) $(LIBXXH) $(DESTDIR)$(LIBDIR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT)
@ -494,20 +536,30 @@ install: lib libxxhash.pc xxhsum ## install libraries, CLI, links and man page
ifeq ($(DISPATCH),1)
$(Q)$(INSTALL_DATA) xxh_x86dispatch.h $(DESTDIR)$(INCLUDEDIR)
endif
install_libxxhash.pc: libxxhash.pc
@echo Installing pkgconfig
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
$(Q)$(INSTALL_DIR) $(DESTDIR)$(PKGCONFIGDIR)/
$(Q)$(INSTALL_DATA) libxxhash.pc $(DESTDIR)$(PKGCONFIGDIR)/
install_xxhsum: xxhsum
@echo Installing xxhsum
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
$(Q)$(INSTALL_DIR) $(DESTDIR)$(BINDIR)/
$(Q)$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh128sum
install_man:
@echo Installing man pages
$(Q)$(INSTALL_DIR) $(DESTDIR)$(MANDIR)/
$(Q)$(INSTALL_DATA) $(MAN) $(DESTDIR)$(MANDIR)/xxhsum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh128sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh128sum.1
.PHONY: install
install: install_libxxhash.a install_libxxhash install_libxxhash.pc install_xxhsum install_man ## install libraries, CLI, links and man page
@echo xxhash installation completed
.PHONY: uninstall

50
deps/xxHash/README.md vendored
View file

@ -9,6 +9,7 @@ Code is highly portable, and hashes are identical across all platforms (little /
|Branch |Status |
|------------|---------|
|release | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=release)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Arelease+) |
|dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) |
@ -110,35 +111,42 @@ The following macros can be set at compilation time to modify libxxhash's behavi
It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses).
This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms.
- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for AVX2, and `-mavx512f` for AVX512).
- `XXH_NO_PREFETCH` : disable prefetching. XXH3 only.
- `XXH_PREFETCH_DIST` : select prefecting distance. XXH3 only.
- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only.
- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only.
- `XXH_NO_STREAM`: Disables the streaming API, limiting it to single shot variants only.
- `XXH_SIZE_OPT`: `0`: default, optimize for speed
`1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization
`2`: makes code as small as possible, performance may cry
- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size.
Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not.
This is very useful when optimizing for smallest binary size,
and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang.
This may also increase performance depending on compiler and architecture.
- `XXH_REROLL`: Reduces the size of the generated code by not unrolling some loops.
Impact on performance may vary, depending on platform and algorithm.
- `XXH_ACCEPT_NULL_INPUT_POINTER`: if set to `1`, when input is a `NULL` pointer,
xxHash'd result is the same as a zero-length input
(instead of a dereference segfault).
Adds one branch at the beginning of each hash.
- `XXH_STATIC_LINKING_ONLY`: gives access to the state declaration for static allocation.
- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump.
This is generally undesirable for performance, especially when hashing inputs of random sizes.
But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default.
- `XXH_NO_STDLIB`: Disable invocation of `<stdlib.h>` functions, notably `malloc()` and `free()`.
`libxxhash`'s `XXH*_createState()` will always fail and return `NULL`.
But one-shot hashing (like `XXH32()`) or streaming using statically allocated states
still work as expected.
This build flag is useful for embedded environments without dynamic allocation.
- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation.
Incompatible with dynamic linking, due to risks of ABI changes.
- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary.
Useful to reduce binary size, notably for applications which do not use `XXH3`.
- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (XXH3 and XXH64). Only XXH32 will be compiled.
Useful for targets (architectures and compilers) without 64-bit support.
- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors.
- `XXH_CPU_LITTLE_ENDIAN`: By default, endianess is determined by a runtime test resolved at compile time.
- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time.
If, for some reason, the compiler cannot simplify the runtime test, it can cost performance.
It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1.
Setting it to 0 states big-endian.
- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements.
This (slightly) slows down execution, but may help finding bugs during debugging sessions.
For the Command Line Interface `xxhsum`, the following environment variables can also be set :
When compiling the Command Line Interface `xxhsum` with `make`, the following environment variables can also be set :
- `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems.
### Building xxHash - Using vcpkg
You can download and install xxHash using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager:
@ -151,6 +159,24 @@ You can download and install xxHash using the [vcpkg](https://github.com/Microso
The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
### Building and Using xxHash - tipi.build
You can work on xxHash and depend on it in your [tipi.build](https://tipi.build) projects by adding the following entry to your `.tipi/deps`:
```json
{
"Cyan4973/xxHash": { "@": "v0.8.1" }
}
```
An example of such usage can be found in the `/cli` folder of this project which, if built as root project will depend on the release `v0.8.1` of xxHash
To contribute to xxHash itself use tipi.build on this repository (change the target name appropriately to `linux` or `macos` or `windows`):
```bash
tipi . -t <target> --test all
```
### Example

20
deps/xxHash/cli/xsum_arch.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -79,7 +79,7 @@
#endif
/* makes the next part easier */
#if defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
#if (defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) && !defined(_M_ARM64EC)
# define XSUM_ARCH_X64 1
# define XSUM_ARCH_X86 "x86_64"
#elif defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP)
@ -102,11 +102,17 @@
# else
# define XSUM_ARCH XSUM_ARCH_X86
# endif
#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)
#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
# define XSUM_ARCH "aarch64 + NEON"
#elif defined(__arm__) || defined(__thumb__) || defined(__thumb2__) || defined(_M_ARM)
/* ARM has a lot of different features that can change xxHash significantly. */
# if defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7))
# ifdef __ARM_ARCH
# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(__ARM_ARCH)
# else
# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(_M_ARM)
# endif
# if defined(_M_ARM) /* windows arm is always thumb-2 */ \
|| defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7))
# define XSUM_ARCH_THUMB " Thumb-2"
# elif defined(__thumb__)
# define XSUM_ARCH_THUMB " Thumb-1"
@ -114,17 +120,17 @@
# define XSUM_ARCH_THUMB ""
# endif
/* ARMv7 has unaligned by default */
# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARMV7VE)
# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARM)
# define XSUM_ARCH_UNALIGNED " + unaligned"
# else
# define XSUM_ARCH_UNALIGNED ""
# endif
# if defined(__ARM_NEON) || defined(__ARM_NEON__)
# if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM)
# define XSUM_ARCH_NEON " + NEON"
# else
# define XSUM_ARCH_NEON ""
# endif
# define XSUM_ARCH "ARMv" XSUM_EXPAND_AND_QUOTE(__ARM_ARCH) XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED
# define XSUM_ARCH "ARMv" XSUM_ARCH_ARM_VER XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED
#elif defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)
# if defined(__GNUC__) && defined(__POWER9_VECTOR__)
# define XSUM_ARCH "ppc64 + POWER9 vector"

436
deps/xxHash/cli/xsum_bench.c vendored Normal file
View file

@ -0,0 +1,436 @@
/*
* xsum_bench - Benchmark functions for xxhsum
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include "xsum_output.h" /* XSUM_logLevel */
#include "xsum_bench.h"
#include "xsum_sanity_check.h" /* XSUM_fillTestBuffer */
#include "xsum_os_specific.h" /* XSUM_getFileSize */
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
#endif
#include "../xxhash.h"
#ifdef XXHSUM_DISPATCH
# include "../xxh_x86dispatch.h" /* activate _dispatch() redirectors */
#endif
#include <stdlib.h> /* malloc, free */
#include <assert.h>
#include <string.h> /* strlen, memcpy */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include <errno.h> /* errno */
#define TIMELOOP_S 1
#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* target timing per iteration */
#define TIMELOOP_MIN (TIMELOOP / 2) /* minimum timing to validate a result */
#define MAX_MEM (2 GB - 64 MB)
static clock_t XSUM_clockSpan( clock_t start )
{
return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */
}
static size_t XSUM_findMaxMem(XSUM_U64 requiredMem)
{
size_t const step = 64 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += 2*step;
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
while (!testmem) {
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
testmem = malloc ((size_t)requiredMem);
}
free (testmem);
/* keep some space available */
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
return (size_t)requiredMem;
}
/*
* A secret buffer used for benchmarking XXH3's withSecret variants.
*
* In order for the bench to be realistic, the secret buffer would need to be
* pre-generated.
*
* Adding a pointer to the parameter list would be messy.
*/
static XSUM_U8 g_benchSecretBuf[XXH3_SECRET_SIZE_MIN];
/*
* Wrappers for the benchmark.
*
* If you would like to add other hashes to the bench, create a wrapper and add
* it to the g_hashesToBench table. It will automatically be added.
*/
typedef XSUM_U32 (*hashFunction)(const void* buffer, size_t bufferSize, XSUM_U32 seed);
static XSUM_U32 localXXH32(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return XXH32(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH32_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH32_state_t state;
(void)seed;
XXH32_reset(&state, seed);
XXH32_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH32_digest(&state);
}
static XSUM_U32 localXXH64(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)XXH64(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH64_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH64_state_t state;
(void)seed;
XXH64_reset(&state, seed);
XXH64_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH64_digest(&state);
}
static XSUM_U32 localXXH3_64b(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)XXH3_64bits(buffer, bufferSize);
}
static XSUM_U32 localXXH3_64b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)XXH3_64bits_withSeed(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH3_64b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)XXH3_64bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf));
}
static XSUM_U32 localXXH3_128b(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)(XXH3_128bits(buffer, bufferSize).low64);
}
static XSUM_U32 localXXH3_128b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)(XXH3_128bits_withSeed(buffer, bufferSize, seed).low64);
}
static XSUM_U32 localXXH3_128b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)(XXH3_128bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)).low64);
}
static XSUM_U32 localXXH3_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
(void)seed;
XXH3_64bits_reset(&state);
XXH3_64bits_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH3_64bits_digest(&state);
}
static XSUM_U32 localXXH3_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
XXH3_INITSTATE(&state);
XXH3_64bits_reset_withSeed(&state, (XXH64_hash_t)seed);
XXH3_64bits_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH3_64bits_digest(&state);
}
static XSUM_U32 localXXH128_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
(void)seed;
XXH3_128bits_reset(&state);
XXH3_128bits_update(&state, buffer, bufferSize);
return (XSUM_U32)(XXH3_128bits_digest(&state).low64);
}
static XSUM_U32 localXXH128_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
XXH3_INITSTATE(&state);
XXH3_128bits_reset_withSeed(&state, (XXH64_hash_t)seed);
XXH3_128bits_update(&state, buffer, bufferSize);
return (XSUM_U32)(XXH3_128bits_digest(&state).low64);
}
typedef struct {
const char* name;
hashFunction func;
} hashInfo;
static const hashInfo g_hashesToBench[] = {
{ "XXH32", &localXXH32 },
{ "XXH64", &localXXH64 },
{ "XXH3_64b", &localXXH3_64b },
{ "XXH3_64b w/seed", &localXXH3_64b_seeded },
{ "XXH3_64b w/secret", &localXXH3_64b_secret },
{ "XXH128", &localXXH3_128b },
{ "XXH128 w/seed", &localXXH3_128b_seeded },
{ "XXH128 w/secret", &localXXH3_128b_secret },
{ "XXH32_stream", &localXXH32_stream },
{ "XXH64_stream", &localXXH64_stream },
{ "XXH3_stream", &localXXH3_stream },
{ "XXH3_stream w/seed",&localXXH3_stream_seeded },
{ "XXH128_stream", &localXXH128_stream },
{ "XXH128_stream w/seed",&localXXH128_stream_seeded },
};
#define NB_HASHFUNC (sizeof(g_hashesToBench) / sizeof(*g_hashesToBench))
#define NB_TESTFUNC (1 + 2 * NB_HASHFUNC)
int const g_nbTestFunctions = NB_TESTFUNC;
char g_testIDs[NB_TESTFUNC] = { 0 };
const char k_testIDs_default[NB_TESTFUNC] = { 0,
1 /*XXH32*/, 0,
1 /*XXH64*/, 0,
1 /*XXH3*/, 0, 0, 0, 0, 0,
1 /*XXH128*/ };
int g_nbIterations = NBLOOPS_DEFAULT;
#define HASHNAME_MAX 29
static void XSUM_benchHash(hashFunction h, const char* hName, int testID,
const void* buffer, size_t bufferSize)
{
XSUM_U32 nbh_perIteration = (XSUM_U32)((300 MB) / (bufferSize+1)) + 1; /* first iteration conservatively aims for 300 MB/s */
int iterationNb, nbIterations = g_nbIterations + !g_nbIterations /* min 1 */;
double fastestH = 100000000.;
assert(HASHNAME_MAX > 2);
XSUM_logVerbose(2, "\r%80s\r", ""); /* Clean display line */
for (iterationNb = 1; iterationNb <= nbIterations; iterationNb++) {
XSUM_U32 r=0;
clock_t cStart;
XSUM_logVerbose(2, "%2i-%-*.*s : %10u ->\r",
iterationNb,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize);
cStart = clock();
while (clock() == cStart); /* starts clock() at its exact beginning */
cStart = clock();
{ XSUM_U32 u;
for (u=0; u<nbh_perIteration; u++)
r += h(buffer, bufferSize, u);
}
if (r==0) XSUM_logVerbose(3,".\r"); /* do something with r to defeat compiler "optimizing" hash away */
{ clock_t const nbTicks = XSUM_clockSpan(cStart);
double const ticksPerHash = ((double)nbTicks / TIMELOOP) / nbh_perIteration;
/*
* clock() is the only decent portable timer, but it isn't very
* precise.
*
* Sometimes, this lack of precision is enough that the benchmark
* finishes before there are enough ticks to get a meaningful result.
*
* For example, on a Core 2 Duo (without any sort of Turbo Boost),
* the imprecise timer caused peculiar results like so:
*
* XXH3_64b 4800.0 MB/s // conveniently even
* XXH3_64b unaligned 4800.0 MB/s
* XXH3_64b seeded 9600.0 MB/s // magical 2x speedup?!
* XXH3_64b seeded unaligned 4800.0 MB/s
*
* If we sense a suspiciously low number of ticks, we increase the
* iterations until we can get something meaningful.
*/
if (nbTicks < TIMELOOP_MIN) {
/* Not enough time spent in benchmarking, risk of rounding bias */
if (nbTicks == 0) { /* faster than resolution timer */
nbh_perIteration *= 100;
} else {
/*
* update nbh_perIteration so that the next round lasts
* approximately 1 second.
*/
double nbh_perSecond = (1 / ticksPerHash) + 1;
if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */
nbh_perIteration = (XSUM_U32)nbh_perSecond;
}
/* g_nbIterations==0 => quick evaluation, no claim of accuracy */
if (g_nbIterations>0) {
iterationNb--; /* new round for a more accurate speed evaluation */
continue;
}
}
if (ticksPerHash < fastestH) fastestH = ticksPerHash;
if (fastestH>0.) { /* avoid div by zero */
XSUM_logVerbose(2, "%2i-%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \r",
iterationNb,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize,
(double)1 / fastestH,
((double)bufferSize / (1 MB)) / fastestH);
} }
{ double nbh_perSecond = (1 / fastestH) + 1;
if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */
nbh_perIteration = (XSUM_U32)nbh_perSecond;
}
}
XSUM_logVerbose(1, "%2i#%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \n",
testID,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize,
(double)1 / fastestH,
((double)bufferSize / (1 MB)) / fastestH);
if (XSUM_logLevel<1)
XSUM_logVerbose(0, "%u, ", (unsigned)((double)1 / fastestH));
}
/*
* Allocates a string containing s1 and s2 concatenated. Acts like strdup.
* The result must be freed.
*/
static char* XSUM_strcatDup(const char* s1, const char* s2)
{
assert(s1 != NULL);
assert(s2 != NULL);
{ size_t len1 = strlen(s1);
size_t len2 = strlen(s2);
char* buf = (char*)malloc(len1 + len2 + 1);
if (buf != NULL) {
/* strcpy(buf, s1) */
memcpy(buf, s1, len1);
/* strcat(buf, s2) */
memcpy(buf + len1, s2, len2 + 1);
}
return buf;
}
}
/*!
* XSUM_benchMem():
* buffer: Must be 16-byte aligned.
* The real allocated size of buffer is supposed to be >= (bufferSize+3).
* returns: 0 on success, 1 if error (invalid mode selected)
*/
static void XSUM_benchMem(const void* buffer, size_t bufferSize)
{
assert((((size_t)buffer) & 15) == 0); /* ensure alignment */
XSUM_fillTestBuffer(g_benchSecretBuf, sizeof(g_benchSecretBuf));
{ int i;
for (i = 1; i < (int)NB_TESTFUNC; i++) {
int const hashFuncID = (i-1) / 2;
assert(g_hashesToBench[hashFuncID].name != NULL);
if (g_testIDs[i] == 0) continue;
/* aligned */
if ((i % 2) == 1) {
XSUM_benchHash(g_hashesToBench[hashFuncID].func, g_hashesToBench[hashFuncID].name, i, buffer, bufferSize);
}
/* unaligned */
if ((i % 2) == 0) {
/* Append "unaligned". */
char* const hashNameBuf = XSUM_strcatDup(g_hashesToBench[hashFuncID].name, " unaligned");
assert(hashNameBuf != NULL);
XSUM_benchHash(g_hashesToBench[hashFuncID].func, hashNameBuf, i, ((const char*)buffer)+3, bufferSize);
free(hashNameBuf);
}
} }
}
static size_t XSUM_selectBenchedSize(const char* fileName)
{
XSUM_U64 const inFileSize = XSUM_getFileSize(fileName);
size_t benchedSize = (size_t) XSUM_findMaxMem(inFileSize);
if ((XSUM_U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize) {
XSUM_log("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20));
}
return benchedSize;
}
int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles)
{
int fileIdx;
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
const char* const inFileName = fileNamesTable[fileIdx];
assert(inFileName != NULL);
{ FILE* const inFile = XSUM_fopen( inFileName, "rb" );
size_t const benchedSize = XSUM_selectBenchedSize(inFileName);
char* const buffer = (char*)calloc(benchedSize+16+3, 1);
void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes */
/* Checks */
if (inFile==NULL){
XSUM_log("Error: Could not open '%s': %s.\n", inFileName, strerror(errno));
free(buffer);
exit(11);
}
if(!buffer) {
XSUM_log("\nError: Out of memory.\n");
fclose(inFile);
exit(12);
}
/* Fill input buffer */
{ size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile);
fclose(inFile);
if(readSize != benchedSize) {
XSUM_log("\nError: Could not read '%s': %s.\n", inFileName, strerror(errno));
free(buffer);
exit(13);
} }
/* bench */
XSUM_benchMem(alignedBuffer, benchedSize);
free(buffer);
} }
return 0;
}
int XSUM_benchInternal(size_t keySize)
{
void* const buffer = calloc(keySize+16+3, 1);
if (buffer == NULL) {
XSUM_log("\nError: Out of memory.\n");
exit(12);
}
{ const void* const alignedBuffer = ((char*)buffer+15) - (((size_t)((char*)buffer+15)) & 0xF); /* align on next 16 bytes */
/* bench */
XSUM_logVerbose(1, "Sample of ");
if (keySize > 10 KB) {
XSUM_logVerbose(1, "%u KB", (unsigned)(keySize >> 10));
} else {
XSUM_logVerbose(1, "%u bytes", (unsigned)keySize);
}
XSUM_logVerbose(1, "... \n");
XSUM_benchMem(alignedBuffer, keySize);
free(buffer);
}
return 0;
}

51
deps/xxHash/cli/xsum_bench.h vendored Normal file
View file

@ -0,0 +1,51 @@
/*
* xsum_bench - Benchmark functions for xxhsum
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* You can contact the author at:
* - xxHash homepage: https://www.xxhash.com
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#ifndef XSUM_BENCH_H
#define XSUM_BENCH_H
#include <stddef.h> /* size_t */
#define NBLOOPS_DEFAULT 3 /* Default number of benchmark iterations */
extern int const g_nbTestFunctions;
extern char g_testIDs[]; /* size : g_nbTestFunctions */
extern const char k_testIDs_default[];
extern int g_nbIterations;
int XSUM_benchInternal(size_t keySize);
int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles);
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /* XSUM_BENCH_H */

11
deps/xxHash/cli/xsum_config.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -202,4 +202,13 @@
typedef unsigned long long XSUM_U64;
#endif /* not C++/C99 */
/* ***************************
* Common constants
* ***************************/
#define KB *( 1<<10)
#define MB *( 1<<20)
#define GB *(1U<<30)
#endif /* XSUM_CONFIG_H */

18
deps/xxHash/cli/xsum_os_specific.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -23,12 +23,7 @@
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include "xsum_config.h"
#include "xsum_os_specific.h"
#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <sys/types.h> /* struct stat / __wstat64 */
#include "xsum_os_specific.h" /* XSUM_API */
#include <sys/stat.h> /* stat() / _stat64() */
/*
@ -47,7 +42,8 @@
#if (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \
|| (XSUM_PLATFORM_POSIX_VERSION >= 200112L) \
|| defined(__DJGPP__) \
|| defined(__MSYS__)
|| defined(__MSYS__) \
|| defined(__HAIKU__)
# include <unistd.h> /* isatty */
# define XSUM_IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2)
@ -112,7 +108,7 @@ static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf)
}
#ifndef XSUM_NO_MAIN
int main(int argc, char* argv[])
int main(int argc, const char* argv[])
{
return XSUM_main(argc, argv);
}
@ -383,7 +379,7 @@ static int XSUM_wmain(int argc, wchar_t* utf16_argv[])
setvbuf(stderr, NULL, _IONBF, 0);
/* Call our real main function */
ret = XSUM_main(argc, utf8_argv);
ret = XSUM_main(argc, (void*)utf8_argv);
/* Cleanup */
XSUM_freeArgv(argc, utf8_argv);
@ -439,7 +435,7 @@ int __cdecl __wgetmainargs(
_startupinfo* StartInfo
);
int main(int ansi_argc, char* ansi_argv[])
int main(int ansi_argc, const char* ansi_argv[])
{
int utf16_argc;
wchar_t** utf16_argv;

4
deps/xxHash/cli/xsum_os_specific.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -39,7 +39,7 @@ extern "C" {
*
* Functions like main(), but is passed UTF-8 arguments even on Windows.
*/
XSUM_API int XSUM_main(int argc, char* argv[]);
XSUM_API int XSUM_main(int argc, const char* argv[]);
/*
* Returns whether stream is a console.

5
deps/xxHash/cli/xsum_output.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -24,8 +24,7 @@
*/
#include "xsum_output.h"
#include "xsum_os_specific.h"
#include <stdio.h>
#include "xsum_os_specific.h" /* XSUM_API */
int XSUM_logLevel = 2;

2
deps/xxHash/cli/xsum_output.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*

128
deps/xxHash/cli/xsum_sanity_check.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -23,17 +23,17 @@
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include "xsum_config.h"
#include "xsum_sanity_check.h"
#include "xsum_output.h"
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include "xsum_output.h" /* XSUM_log */
#ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
#endif
#include "../xxhash.h"
#include <stdlib.h> /* exit */
#include <assert.h>
#include <string.h> /* memcmp */
/* use #define to make them constant, required for initialization */
#define PRIME32 2654435761U
#define PRIME64 11400714785074694797ULL
@ -60,7 +60,7 @@ XSUM_API void XSUM_fillTestBuffer(XSUM_U8* buffer, size_t len)
/* ************************************************
* Self-test:
* ensure results consistency accross platforms
* ensure results consistency across platforms
*********************************************** */
#if XSUM_NO_TESTS
XSUM_API void XSUM_sanityCheck(void)
@ -90,9 +90,10 @@ typedef struct {
XXH128_hash_t Nresult;
} XSUM_testdata128_t;
#define SECRET_SAMPLE_NBBYTES 4
#define SECRET_SAMPLE_NBBYTES 5
typedef struct {
XSUM_U32 len;
XSUM_U32 seedLen;
XSUM_U32 secretLen;
XSUM_U8 byte[SECRET_SAMPLE_NBBYTES];
} XSUM_testdata_sample_t;
@ -213,11 +214,12 @@ static const XSUM_testdata128_t XSUM_XXH128_withSecret_testdata[] = {
{ 12, 0, { 0xAF82F6EBA263D7D8ULL, 0x90A3C2D839F57D0FULL } } /* 9 - 16 */
};
#define SECRET_SIZE_MAX 9867
static const XSUM_testdata_sample_t XSUM_XXH3_generateSecret_testdata[] = {
{ 0, { 0xB8, 0x26, 0x83, 0x7E } },
{ 1, { 0xA6, 0x16, 0x06, 0x7B } },
{ XXH3_SECRET_SIZE_MIN - 1, { 0xDA, 0x2A, 0x12, 0x11 } },
{ XXH3_SECRET_DEFAULT_SIZE + 500, { 0x7E, 0x48, 0x0C, 0xA7 } }
{ 0, 192, { 0xE7, 0x8C, 0x77, 0x77, 0x00 } },
{ 1, 240, { 0x2B, 0x3E, 0xDE, 0xC1, 0x00 } },
{ XXH3_SECRET_SIZE_MIN - 1, 277, { 0xE8, 0x39, 0x6C, 0xCC, 0x7B } },
{ XXH3_SECRET_DEFAULT_SIZE + 500, SECRET_SIZE_MAX, { 0xD6, 0x1C, 0x41, 0x17, 0xB3 } }
};
static void XSUM_checkResult32(XXH32_hash_t r1, XXH32_hash_t r2)
@ -378,6 +380,16 @@ static void XSUM_testXXH3(const void* data, const XSUM_testdata64_t* testData)
XSUM_checkResult64(Dresult, Nresult);
}
/* check that the combination of
* XXH3_generateSecret_fromSeed() and XXH3_64bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
{ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
XSUM_checkResult64(Dresult, Nresult);
} }
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
assert(state != NULL);
@ -398,10 +410,29 @@ static void XSUM_testXXH3(const void* data, const XSUM_testdata64_t* testData)
(void)XXH3_64bits_update(state, ((const char*)data)+pos, 1);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
}
/* check that streaming with a combination of
* XXH3_generateSecret_fromSeed() and XXH3_64bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
/* single ingestion */
(void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_64bits_update(state, data, len);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
}
XXH3_freeState(state);
}
}
#ifndef XXH3_MIDSIZE_MAX
# define XXH3_MIDSIZE_MAX 240
#endif
static void XSUM_testXXH3_withSecret(const void* data, const void* secret,
size_t secretSize, const XSUM_testdata64_t* testData)
{
@ -417,6 +448,13 @@ static void XSUM_testXXH3_withSecret(const void* data, const void* secret,
XSUM_checkResult64(Dresult, Nresult);
}
/* check that XXH3_64bits_withSecretandSeed()
* results in exactly the same return value as XXH3_64bits_withSecret() */
if (len > XXH3_MIDSIZE_MAX)
{ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, secretSize, 0);
XSUM_checkResult64(Dresult, Nresult);
}
/* streaming API test */
{ XXH3_state_t *state = XXH3_createState();
assert(state != NULL);
@ -436,6 +474,16 @@ static void XSUM_testXXH3_withSecret(const void* data, const void* secret,
(void)XXH3_64bits_update(state, ((const char*)data)+pos, 1);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
}
/* check that XXH3_64bits_reset_withSecretandSeed()
* results in exactly the same return value as XXH3_64bits_reset_withSecret() */
if (len > XXH3_MIDSIZE_MAX) {
/* single ingestion */
(void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 0);
(void)XXH3_64bits_update(state, data, len);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
}
XXH3_freeState(state);
}
}
@ -466,6 +514,16 @@ static void XSUM_testXXH128(const void* data, const XSUM_testdata128_t* testData
XSUM_checkResult128(Dresult, Nresult);
}
/* check that the combination of
* XXH3_generateSecret_fromSeed() and XXH3_128bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
XSUM_checkResult128(Dresult, Nresult);
} }
/* streaming API test */
{ XXH3_state_t *state = XXH3_createState();
assert(state != NULL);
@ -487,6 +545,19 @@ static void XSUM_testXXH128(const void* data, const XSUM_testdata128_t* testData
(void)XXH3_128bits_update(state, ((const char*)data)+pos, 1);
XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
}
/* check that streaming with a combination of
* XXH3_generateSecret_fromSeed() and XXH3_128bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_128bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
/* single ingestion */
(void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_128bits_update(state, data, len);
XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
}
XXH3_freeState(state);
}
}
@ -504,6 +575,13 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz
XSUM_checkResult128(Dresult, Nresult);
}
/* check that XXH3_128bits_withSecretandSeed()
* results in exactly the same return value as XXH3_128bits_withSecret() */
if (len > XXH3_MIDSIZE_MAX)
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, secretSize, 0);
XSUM_checkResult128(Dresult, Nresult);
}
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
assert(state != NULL);
@ -523,6 +601,16 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz
(void)XXH3_128bits_update(state, ((const char*)data)+pos, 1);
XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
}
/* check that XXH3_128bits_reset_withSecretandSeed()
* results in exactly the same return value as XXH3_128bits_reset_withSecret() */
if (len > XXH3_MIDSIZE_MAX) {
/* single ingestion */
(void)XXH3_128bits_reset_withSecretandSeed(state, secret, secretSize, 0);
(void)XXH3_128bits_update(state, data, len);
XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
}
XXH3_freeState(state);
}
}
@ -530,20 +618,21 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz
static void XSUM_testSecretGenerator(const void* customSeed, const XSUM_testdata_sample_t* testData)
{
static int nbTests = 1;
const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191};
XSUM_U8 secretBuffer[XXH3_SECRET_DEFAULT_SIZE] = {0};
const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191, 241 }; /* position of sampled bytes */
XSUM_U8 secretBuffer[SECRET_SIZE_MAX] = {0};
XSUM_U8 samples[SECRET_SAMPLE_NBBYTES];
int i;
XXH3_generateSecret(secretBuffer, customSeed, testData->len);
assert(testData->secretLen <= SECRET_SIZE_MAX);
XXH3_generateSecret(secretBuffer, testData->secretLen, customSeed, testData->seedLen);
for (i=0; i<SECRET_SAMPLE_NBBYTES; i++) {
samples[i] = secretBuffer[sampleIndex[i]];
}
if (memcmp(samples, testData->byte, sizeof(testData->byte))) {
XSUM_log("\rError: Secret generation test %i: Internal sanity check failed. \n", nbTests);
XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n",
samples[0], samples[1], samples[2], samples[3],
testData->byte[0], testData->byte[1], testData->byte[2], testData->byte[3] );
XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n",
samples[0], samples[1], samples[2], samples[3], samples[4],
testData->byte[0], testData->byte[1], testData->byte[2], testData->byte[3], testData->byte[4] );
exit(1);
}
nbTests++;
@ -592,6 +681,7 @@ XSUM_API void XSUM_sanityCheck(void)
}
/* secret generator */
for (i = 0; i < (sizeof(XSUM_XXH3_generateSecret_testdata)/sizeof(XSUM_XXH3_generateSecret_testdata[0])); i++) {
assert(XSUM_XXH3_generateSecret_testdata[i].seedLen <= SANITY_BUFFER_SIZE);
XSUM_testSecretGenerator(sanityBuffer, &XSUM_XXH3_generateSecret_testdata[i]);
}

2
deps/xxHash/cli/xsum_sanity_check.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*

69
deps/xxHash/cli/xxhsum.1 vendored
View file

@ -1,165 +1,106 @@
.
.TH "XXHSUM" "1" "July 2020" "xxhsum 0.7.4" "User Commands"
.
.TH "XXHSUM" "1" "December 2021" "xxhsum 0.8.1" "User Commands"
.SH "NAME"
\fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums
.
.SH "SYNOPSIS"
\fBxxhsum [<OPTION>] \.\.\. [<FILE>] \.\.\.\fR \fBxxhsum \-b [<OPTION>] \.\.\.\fR
.
\fBxxhsum [<OPTION>] \|\.\|\.\|\. [<FILE>] \|\.\|\.\|\.\fR \fBxxhsum \-b [<OPTION>] \|\.\|\.\|\.\fR
.P
\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR
.
\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR, \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR, \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR
.SH "DESCRIPTION"
Print or check xxHash (32, 64 or 128 bits) checksums\. When no \fIFILE\fR, read standard input, except if it\'s the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it\'s the console\.
.
.P
\fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are: \fBxxhsum\fR doesn\'t have text/binary mode switch (\fB\-b\fR, \fB\-t\fR); \fBxxhsum\fR always treats files as binary file; \fBxxhsum\fR has a hash bit width switch (\fB\-H\fR);
.
\fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are: \fBxxhsum\fR doesn\'t have text/binary mode switch (\fB\-b\fR, \fB\-t\fR); \fBxxhsum\fR always treats files as binary file; \fBxxhsum\fR has a hash selection switch (\fB\-H\fR);
.P
As xxHash is a fast non\-cryptographic checksum algorithm, \fBxxhsum\fR should not be used for security related purposes\.
.
.P
\fBxxhsum \-b\fR invokes benchmark mode\. See \fIOPTIONS\fR and \fIEXAMPLES\fR for details\.
.
.SH "OPTIONS"
.
.TP
\fB\-V\fR, \fB\-\-version\fR
Displays xxhsum version and exits
.
.TP
\fB\-H\fR\fIHASHTYPE\fR
Hash selection\. \fIHASHTYPE\fR means \fB0\fR=32bits, \fB1\fR=64bits, \fB2\fR=128bits\. Alternatively, \fIHASHTYPE\fR \fB32\fR=32bits, \fB64\fR=64bits, \fB128\fR=128bits\. Default value is \fB1\fR (64bits)
.
Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Note that \fB\-H3\fR triggers \fB\-\-tag\fR, which can\'t be skipped (this is to reduce risks of confusion with \fB\-H2\fR (\fBXXH64\fR))\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (64bits)
.TP
\fB\-\-tag\fR
Output in the BSD style\.
.
.TP
\fB\-\-little\-endian\fR
Set output hexadecimal checksum value as little endian convention\. By default, value is displayed as big endian\.
.
.TP
\fB\-h\fR, \fB\-\-help\fR
Displays help and exits
.
.P
\fBThe following four options are useful only when verifying checksums (\fB\-c\fR)\fR
.
.TP
\fB\-c\fR, \fB\-\-check\fR \fIFILE\fR
Read xxHash sums from \fIFILE\fR and check them
.
.TP
\fB\-q\fR, \fB\-\-quiet\fR
Don\'t print OK for each successfully verified file
.
.TP
\fB\-\-strict\fR
Return an error code if any line in the file is invalid, not just if some checksums are wrong\. This policy is disabled by default, though UI will prompt an informational message if any line in the file is detected invalid\.
.
.TP
\fB\-\-status\fR
Don\'t output anything\. Status code shows success\.
.
.TP
\fB\-w\fR, \fB\-\-warn\fR
Emit a warning message about each improperly formatted checksum line\.
.
.P
\fBThe following options are useful only benchmark purpose\fR
.
.TP
\fB\-b\fR
Benchmark mode\. See \fIEXAMPLES\fR for details\.
.
.TP
\fB\-b#\fR
Specify ID of variant to be tested\. Multiple variants can be selected, separated by a \',\' comma\.
.
.TP
\fB\-B\fR\fIBLOCKSIZE\fR
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode\'s test data block size in bytes\. Default value is 102400
.
.TP
\fB\-i\fR\fIITERATIONS\fR
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIITERATIONS\fR specifies number of iterations in benchmark\. Single iteration lasts approximately 1000 milliseconds\. Default value is 3
.
.SH "EXIT STATUS"
\fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn\'t be read or doesn\'t have the same checksum as the \fB\-c\fR option\.
.
.SH "EXAMPLES"
Output xxHash (64bit) checksum values of specific files to standard output
.
.IP "" 4
.
.nf
$ xxhsum \-H1 foo bar baz
.
.fi
.
.IP "" 0
.
.P
Output xxHash (32bit and 64bit) checksum values of specific files to standard output, and redirect it to \fBxyz\.xxh32\fR and \fBqux\.xxh64\fR
.
.IP "" 4
.
.nf
$ xxhsum \-H0 foo bar baz > xyz\.xxh32
$ xxhsum \-H1 foo bar baz > qux\.xxh64
.
.fi
.
.IP "" 0
.
.P
Read xxHash sums from specific files and check them
.
.IP "" 4
.
.nf
$ xxhsum \-c xyz\.xxh32 qux\.xxh64
.
.fi
.
.IP "" 0
.
.P
Benchmark xxHash algorithm\. By default, \fBxxhsum\fR benchmarks xxHash main variants on a synthetic sample of 100 KB, and print results into standard output\. The first column is the algorithm, the second column is the source data size in bytes, the third column is the number of hashes generated per second (throughput), and finally the last column translates speed in megabytes per second\.
.
.IP "" 4
.
.nf
$ xxhsum \-b
.
.fi
.
.IP "" 0
.
.P
In the following example, the sample to hash is set to 16384 bytes, the variants to be benched are selected by their IDs, and each benchmark test is repeated 10 times, for increased accuracy\.
.
.IP "" 4
.
.nf
$ xxhsum \-b1,2,3 \-i10 \-B16384
.
.fi
.
.IP "" 0
.
.SH "BUGS"
Report bugs at: https://github\.com/Cyan4973/xxHash/issues/
.
.SH "AUTHOR"
Yann Collet
.
.SH "SEE ALSO"
md5sum(1)

16
deps/xxHash/cli/xxhsum.1.md vendored
View file

@ -7,9 +7,9 @@ SYNOPSIS
`xxhsum [<OPTION>] ... [<FILE>] ...`
`xxhsum -b [<OPTION>] ...`
`xxh32sum` is equivalent to `xxhsum -H0`
`xxh64sum` is equivalent to `xxhsum -H1`
`xxh128sum` is equivalent to `xxhsum -H2`
`xxh32sum` is equivalent to `xxhsum -H0`,
`xxh64sum` is equivalent to `xxhsum -H1`,
`xxh128sum` is equivalent to `xxhsum -H2`.
DESCRIPTION
@ -23,7 +23,7 @@ When <FILE> is `-`, read standard input even if it's the console.
Differences are:
`xxhsum` doesn't have text/binary mode switch (`-b`, `-t`);
`xxhsum` always treats files as binary file;
`xxhsum` has a hash bit width switch (`-H`);
`xxhsum` has a hash selection switch (`-H`);
As xxHash is a fast non-cryptographic checksum algorithm,
`xxhsum` should not be used for security related purposes.
@ -37,9 +37,11 @@ OPTIONS
Displays xxhsum version and exits
* `-H`<HASHTYPE>:
Hash selection. <HASHTYPE> means `0`=32bits, `1`=64bits, `2`=128bits.
Alternatively, <HASHTYPE> `32`=32bits, `64`=64bits, `128`=128bits.
Default value is `1` (64bits)
Hash selection. <HASHTYPE> means `0`=XXH32, `1`=XXH64, `2`=XXH128, `3`=XXH3.
Note that `-H3` triggers `--tag`, which can't be skipped
(this is to reduce risks of confusion with `-H2` (`XXH64`)).
Alternatively, <HASHTYPE> `32`=XXH32, `64`=XXH64, `128`=XXH128.
Default value is `1` (XXH64)
* `--tag`:
Output in the BSD style.

676
deps/xxHash/cli/xxhsum.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -30,27 +30,23 @@
*/
/* Transitional headers */
#include "xsum_config.h"
#include "xsum_arch.h"
#include "xsum_os_specific.h"
#include "xsum_output.h"
#include "xsum_sanity_check.h"
#include "xsum_arch.h" /* XSUM_PROGRAM_VERSION */
#include "xsum_os_specific.h" /* XSUM_setBinaryMode */
#include "xsum_output.h" /* XSUM_output */
#include "xsum_sanity_check.h" /* XSUM_sanityCheck */
#include "xsum_bench.h" /* NBLOOPS_DEFAULT */
#ifdef XXH_INLINE_ALL
# include "xsum_os_specific.c"
# include "xsum_output.c"
# include "xsum_sanity_check.c"
# include "xsum_bench.c"
#endif
/* ************************************
* Includes
**************************************/
#include <limits.h>
#include <stdlib.h> /* malloc, calloc, free, exit */
#include <string.h> /* strcmp, memcpy */
#include <stdio.h> /* fprintf, fopen, ftello64, fread, stdin, stdout, _fileno (when present) */
#include <sys/types.h> /* stat, stat64, _stat64 */
#include <sys/stat.h> /* stat, stat64, _stat64 */
#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
#include <string.h> /* strerror, strcmp, memcpy */
#include <assert.h> /* assert */
#include <errno.h> /* errno */
@ -78,23 +74,10 @@ static const char author[] = "Yann Collet";
exename, XSUM_PROGRAM_VERSION, author, \
g_nbBits, XSUM_ARCH, ENDIAN_NAME, XSUM_CC_VERSION
#define KB *( 1<<10)
#define MB *( 1<<20)
#define GB *(1U<<30)
static size_t XSUM_DEFAULT_SAMPLE_SIZE = 100 KB;
#define NBLOOPS 3 /* Default number of benchmark iterations */
#define TIMELOOP_S 1
#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* target timing per iteration */
#define TIMELOOP_MIN (TIMELOOP / 2) /* minimum timing to validate a result */
#define XXHSUM32_DEFAULT_SEED 0 /* Default seed for algo_xxh32 */
#define XXHSUM64_DEFAULT_SEED 0 /* Default seed for algo_xxh64 */
#define MAX_MEM (2 GB - 64 MB)
static const char stdinName[] = "-";
static const char stdinFileName[] = "stdin";
typedef enum { algo_xxh32=0, algo_xxh64=1, algo_xxh128=2 } AlgoSelected;
typedef enum { algo_xxh32=0, algo_xxh64=1, algo_xxh128=2, algo_xxh3=3 } AlgoSelected;
static AlgoSelected g_defaultAlgo = algo_xxh64; /* required within main() & XSUM_usage() */
/* <16 hex char> <SPC> <SPC> <filename> <'\0'>
@ -104,393 +87,111 @@ static AlgoSelected g_defaultAlgo = algo_xxh64; /* required within main() & X
/* Maximum acceptable line length. */
#define MAX_LINE_LENGTH (32 KB)
/* ************************************
* Display macros
**************************************/
static size_t XSUM_DEFAULT_SAMPLE_SIZE = 100 KB;
/* ************************************
* Local variables
**************************************/
static XSUM_U32 g_nbIterations = NBLOOPS;
/* ************************************
* Benchmark Functions
**************************************/
static clock_t XSUM_clockSpan( clock_t start )
{
return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */
/* ********************************************************
* Filename (un)escaping
**********************************************************/
static int XSUM_filenameNeedsEscape(const char* filename) {
return strchr(filename, '\\')
|| strchr(filename, '\n')
|| strchr(filename, '\r');
}
static size_t XSUM_findMaxMem(XSUM_U64 requiredMem)
{
size_t const step = 64 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 26) + 1) << 26);
requiredMem += 2*step;
if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
while (!testmem) {
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
testmem = malloc ((size_t)requiredMem);
static int XSUM_lineNeedsUnescape(const char* line) {
/* Skip white-space characters */
while (*line == ' ' || *line == '\t') {
++line;
}
free (testmem);
/* keep some space available */
if (requiredMem > step) requiredMem -= step;
else requiredMem >>= 1;
return (size_t)requiredMem;
/* Returns true if first non-white-space character is '\\' (0x5c) */
return *line == '\\';
}
/*
* Allocates a string containing s1 and s2 concatenated. Acts like strdup.
* The result must be freed.
*/
static char* XSUM_strcatDup(const char* s1, const char* s2)
{
assert(s1 != NULL);
assert(s2 != NULL);
{ size_t len1 = strlen(s1);
size_t len2 = strlen(s2);
char* buf = (char*)malloc(len1 + len2 + 1);
if (buf != NULL) {
/* strcpy(buf, s1) */
memcpy(buf, s1, len1);
/* strcat(buf, s2) */
memcpy(buf + len1, s2, len2 + 1);
}
return buf;
}
}
/*
* A secret buffer used for benchmarking XXH3's withSecret variants.
*
* In order for the bench to be realistic, the secret buffer would need to be
* pre-generated.
*
* Adding a pointer to the parameter list would be messy.
*/
static XSUM_U8 g_benchSecretBuf[XXH3_SECRET_SIZE_MIN];
/*
* Wrappers for the benchmark.
*
* If you would like to add other hashes to the bench, create a wrapper and add
* it to the g_hashesToBench table. It will automatically be added.
*/
typedef XSUM_U32 (*hashFunction)(const void* buffer, size_t bufferSize, XSUM_U32 seed);
static XSUM_U32 localXXH32(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return XXH32(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH64(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)XXH64(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH3_64b(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)XXH3_64bits(buffer, bufferSize);
}
static XSUM_U32 localXXH3_64b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)XXH3_64bits_withSeed(buffer, bufferSize, seed);
}
static XSUM_U32 localXXH3_64b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)XXH3_64bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf));
}
static XSUM_U32 localXXH3_128b(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)(XXH3_128bits(buffer, bufferSize).low64);
}
static XSUM_U32 localXXH3_128b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
return (XSUM_U32)(XXH3_128bits_withSeed(buffer, bufferSize, seed).low64);
}
static XSUM_U32 localXXH3_128b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
(void)seed;
return (XSUM_U32)(XXH3_128bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)).low64);
}
static XSUM_U32 localXXH3_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
(void)seed;
XXH3_64bits_reset(&state);
XXH3_64bits_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH3_64bits_digest(&state);
}
static XSUM_U32 localXXH3_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
XXH3_INITSTATE(&state);
XXH3_64bits_reset_withSeed(&state, (XXH64_hash_t)seed);
XXH3_64bits_update(&state, buffer, bufferSize);
return (XSUM_U32)XXH3_64bits_digest(&state);
}
static XSUM_U32 localXXH128_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
(void)seed;
XXH3_128bits_reset(&state);
XXH3_128bits_update(&state, buffer, bufferSize);
return (XSUM_U32)(XXH3_128bits_digest(&state).low64);
}
static XSUM_U32 localXXH128_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed)
{
XXH3_state_t state;
XXH3_INITSTATE(&state);
XXH3_128bits_reset_withSeed(&state, (XXH64_hash_t)seed);
XXH3_128bits_update(&state, buffer, bufferSize);
return (XSUM_U32)(XXH3_128bits_digest(&state).low64);
}
typedef struct {
const char* name;
hashFunction func;
} hashInfo;
#define NB_HASHFUNC 12
static const hashInfo g_hashesToBench[NB_HASHFUNC] = {
{ "XXH32", &localXXH32 },
{ "XXH64", &localXXH64 },
{ "XXH3_64b", &localXXH3_64b },
{ "XXH3_64b w/seed", &localXXH3_64b_seeded },
{ "XXH3_64b w/secret", &localXXH3_64b_secret },
{ "XXH128", &localXXH3_128b },
{ "XXH128 w/seed", &localXXH3_128b_seeded },
{ "XXH128 w/secret", &localXXH3_128b_secret },
{ "XXH3_stream", &localXXH3_stream },
{ "XXH3_stream w/seed",&localXXH3_stream_seeded },
{ "XXH128_stream", &localXXH128_stream },
{ "XXH128_stream w/seed",&localXXH128_stream_seeded },
};
#define NB_TESTFUNC (1 + 2 * NB_HASHFUNC)
static char g_testIDs[NB_TESTFUNC] = { 0 };
static const char k_testIDs_default[NB_TESTFUNC] = { 0,
1 /*XXH32*/, 0,
1 /*XXH64*/, 0,
1 /*XXH3*/, 0, 0, 0, 0, 0,
1 /*XXH128*/ };
#define HASHNAME_MAX 29
static void XSUM_benchHash(hashFunction h, const char* hName, int testID,
const void* buffer, size_t bufferSize)
{
XSUM_U32 nbh_perIteration = (XSUM_U32)((300 MB) / (bufferSize+1)) + 1; /* first iteration conservatively aims for 300 MB/s */
unsigned iterationNb, nbIterations = g_nbIterations + !g_nbIterations /* min 1 */;
double fastestH = 100000000.;
assert(HASHNAME_MAX > 2);
XSUM_logVerbose(2, "\r%80s\r", ""); /* Clean display line */
for (iterationNb = 1; iterationNb <= nbIterations; iterationNb++) {
XSUM_U32 r=0;
clock_t cStart;
XSUM_logVerbose(2, "%2u-%-*.*s : %10u ->\r",
iterationNb,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize);
cStart = clock();
while (clock() == cStart); /* starts clock() at its exact beginning */
cStart = clock();
{ XSUM_U32 u;
for (u=0; u<nbh_perIteration; u++)
r += h(buffer, bufferSize, u);
}
if (r==0) XSUM_logVerbose(3,".\r"); /* do something with r to defeat compiler "optimizing" hash away */
{ clock_t const nbTicks = XSUM_clockSpan(cStart);
double const ticksPerHash = ((double)nbTicks / TIMELOOP) / nbh_perIteration;
/*
* clock() is the only decent portable timer, but it isn't very
* precise.
*
* Sometimes, this lack of precision is enough that the benchmark
* finishes before there are enough ticks to get a meaningful result.
*
* For example, on a Core 2 Duo (without any sort of Turbo Boost),
* the imprecise timer caused peculiar results like so:
*
* XXH3_64b 4800.0 MB/s // conveniently even
* XXH3_64b unaligned 4800.0 MB/s
* XXH3_64b seeded 9600.0 MB/s // magical 2x speedup?!
* XXH3_64b seeded unaligned 4800.0 MB/s
*
* If we sense a suspiciously low number of ticks, we increase the
* iterations until we can get something meaningful.
*/
if (nbTicks < TIMELOOP_MIN) {
/* Not enough time spent in benchmarking, risk of rounding bias */
if (nbTicks == 0) { /* faster than resolution timer */
nbh_perIteration *= 100;
} else {
/*
* update nbh_perIteration so that the next round lasts
* approximately 1 second.
*/
double nbh_perSecond = (1 / ticksPerHash) + 1;
if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */
nbh_perIteration = (XSUM_U32)nbh_perSecond;
}
/* g_nbIterations==0 => quick evaluation, no claim of accuracy */
if (g_nbIterations>0) {
iterationNb--; /* new round for a more accurate speed evaluation */
continue;
}
static void XSUM_printFilename(const char* filename, int needsEscape) {
if (!needsEscape) {
XSUM_output("%s", filename);
} else {
const char* p;
for (p = filename; *p != '\0'; ++p) {
switch (*p)
{
case '\n':
XSUM_output("\\n");
break;
case '\r':
XSUM_output("\\r");
break;
case '\\':
XSUM_output("\\\\");
break;
default:
XSUM_output("%c", *p);
break;
}
if (ticksPerHash < fastestH) fastestH = ticksPerHash;
if (fastestH>0.) { /* avoid div by zero */
XSUM_logVerbose(2, "%2u-%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \r",
iterationNb,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize,
(double)1 / fastestH,
((double)bufferSize / (1 MB)) / fastestH);
} }
{ double nbh_perSecond = (1 / fastestH) + 1;
if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */
nbh_perIteration = (XSUM_U32)nbh_perSecond;
}
}
XSUM_logVerbose(1, "%2i#%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \n",
testID,
HASHNAME_MAX, HASHNAME_MAX, hName,
(unsigned)bufferSize,
(double)1 / fastestH,
((double)bufferSize / (1 MB)) / fastestH);
if (XSUM_logLevel<1)
XSUM_logVerbose(0, "%u, ", (unsigned)((double)1 / fastestH));
}
/* Unescape filename in place.
/*!
* XSUM_benchMem():
* buffer: Must be 16-byte aligned.
* The real allocated size of buffer is supposed to be >= (bufferSize+3).
* returns: 0 on success, 1 if error (invalid mode selected)
*/
static void XSUM_benchMem(const void* buffer, size_t bufferSize)
{
assert((((size_t)buffer) & 15) == 0); /* ensure alignment */
XSUM_fillTestBuffer(g_benchSecretBuf, sizeof(g_benchSecretBuf));
{ int i;
for (i = 1; i < NB_TESTFUNC; i++) {
int const hashFuncID = (i-1) / 2;
assert(g_hashesToBench[hashFuncID].name != NULL);
if (g_testIDs[i] == 0) continue;
/* aligned */
if ((i % 2) == 1) {
XSUM_benchHash(g_hashesToBench[hashFuncID].func, g_hashesToBench[hashFuncID].name, i, buffer, bufferSize);
- Replace '\\', 'n' (0x5c, 0x6e) with '\n' (0x0a).
- Replace '\\', 'r' (0x5c, 0x72) with '\r' (0x0d).
- Replace '\\', '\\' (0x5c, 0x5c) with '\\' (0x5c).
- filename may not contain other backslash sequences.
- filename may not ends with backslash.
- filename may not contain NUL (0x00).
Return filename if everything is okay.
Return NULL if something wrong.
*/
static char* XSUM_filenameUnescape(char* filename, size_t filenameLen) {
char *p = filename;
size_t i;
for (i = 0; i < filenameLen; ++i) {
switch (filename[i])
{
case '\\':
++i;
if (i == filenameLen) {
return NULL; /* Don't accept '\\', <EOL> */
}
/* unaligned */
if ((i % 2) == 0) {
/* Append "unaligned". */
char* const hashNameBuf = XSUM_strcatDup(g_hashesToBench[hashFuncID].name, " unaligned");
assert(hashNameBuf != NULL);
XSUM_benchHash(g_hashesToBench[hashFuncID].func, hashNameBuf, i, ((const char*)buffer)+3, bufferSize);
free(hashNameBuf);
switch (filename[i])
{
case 'n':
*p++ = '\n';
break;
case 'r':
*p++ = '\r';
break;
case '\\':
*p++ = '\\';
break;
default:
return NULL; /* Don't accept any other backslash sequence */
}
} }
}
static size_t XSUM_selectBenchedSize(const char* fileName)
{
XSUM_U64 const inFileSize = XSUM_getFileSize(fileName);
size_t benchedSize = (size_t) XSUM_findMaxMem(inFileSize);
if ((XSUM_U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize) {
XSUM_log("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20));
}
return benchedSize;
}
static int XSUM_benchFiles(char*const* fileNamesTable, int nbFiles)
{
int fileIdx;
for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
const char* const inFileName = fileNamesTable[fileIdx];
assert(inFileName != NULL);
{ FILE* const inFile = XSUM_fopen( inFileName, "rb" );
size_t const benchedSize = XSUM_selectBenchedSize(inFileName);
char* const buffer = (char*)calloc(benchedSize+16+3, 1);
void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes */
/* Checks */
if (inFile==NULL){
XSUM_log("Error: Could not open '%s': %s.\n", inFileName, strerror(errno));
free(buffer);
exit(11);
}
if(!buffer) {
XSUM_log("\nError: Out of memory.\n");
fclose(inFile);
exit(12);
}
/* Fill input buffer */
{ size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile);
fclose(inFile);
if(readSize != benchedSize) {
XSUM_log("\nError: Could not read '%s': %s.\n", inFileName, strerror(errno));
free(buffer);
exit(13);
} }
/* bench */
XSUM_benchMem(alignedBuffer, benchedSize);
free(buffer);
} }
return 0;
}
static int XSUM_benchInternal(size_t keySize)
{
void* const buffer = calloc(keySize+16+3, 1);
if (buffer == NULL) {
XSUM_log("\nError: Out of memory.\n");
exit(12);
}
{ const void* const alignedBuffer = ((char*)buffer+15) - (((size_t)((char*)buffer+15)) & 0xF); /* align on next 16 bytes */
/* bench */
XSUM_logVerbose(1, "Sample of ");
if (keySize > 10 KB) {
XSUM_logVerbose(1, "%u KB", (unsigned)(keySize >> 10));
} else {
XSUM_logVerbose(1, "%u bytes", (unsigned)keySize);
break;
case '\0':
return NULL; /* Don't accept NUL (0x00) */
default:
*p++ = filename[i];
break;
}
XSUM_logVerbose(1, "... \n");
XSUM_benchMem(alignedBuffer, keySize);
free(buffer);
}
return 0;
if (p < filename + filenameLen) {
*p = '\0';
}
return filename;
}
/* ********************************************************
* File Hashing
**********************************************************/
#define XXHSUM32_DEFAULT_SEED 0 /* Default seed for algo_xxh32 */
#define XXHSUM64_DEFAULT_SEED 0 /* Default seed for algo_xxh64 */
/* for support of --little-endian display mode */
static void XSUM_display_LittleEndian(const void* ptr, size_t length)
{
@ -509,9 +210,9 @@ static void XSUM_display_BigEndian(const void* ptr, size_t length)
}
typedef union {
XXH32_hash_t xxh32;
XXH64_hash_t xxh64;
XXH128_hash_t xxh128;
XXH32_hash_t hash32;
XXH64_hash_t hash64; /* also for xxh3_64bits */
XXH128_hash_t hash128;
} Multihash;
/*
@ -526,12 +227,12 @@ XSUM_hashStream(FILE* inFile,
{
XXH32_state_t state32;
XXH64_state_t state64;
XXH3_state_t state128;
XXH3_state_t state3;
/* Init */
(void)XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED);
(void)XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED);
(void)XXH3_128bits_reset(&state128);
(void)XXH3_128bits_reset(&state3);
/* Load file & update hash */
{ size_t readSize;
@ -545,7 +246,10 @@ XSUM_hashStream(FILE* inFile,
(void)XXH64_update(&state64, buffer, readSize);
break;
case algo_xxh128:
(void)XXH3_128bits_update(&state128, buffer, readSize);
(void)XXH3_128bits_update(&state3, buffer, readSize);
break;
case algo_xxh3:
(void)XXH3_64bits_update(&state3, buffer, readSize);
break;
default:
assert(0);
@ -560,13 +264,16 @@ XSUM_hashStream(FILE* inFile,
switch(hashType)
{
case algo_xxh32:
finalHash.xxh32 = XXH32_digest(&state32);
finalHash.hash32 = XXH32_digest(&state32);
break;
case algo_xxh64:
finalHash.xxh64 = XXH64_digest(&state64);
finalHash.hash64 = XXH64_digest(&state64);
break;
case algo_xxh128:
finalHash.xxh128 = XXH3_128bits_digest(&state128);
finalHash.hash128 = XXH3_128bits_digest(&state3);
break;
case algo_xxh3:
finalHash.hash64 = XXH3_64bits_digest(&state3);
break;
default:
assert(0);
@ -576,9 +283,9 @@ XSUM_hashStream(FILE* inFile,
}
/* algo_xxh32, algo_xxh64, algo_xxh128 */
static const char* XSUM_algoName[] = { "XXH32", "XXH64", "XXH128" };
static const char* XSUM_algoLE_name[] = { "XXH32_LE", "XXH64_LE", "XXH128_LE" };
static const size_t XSUM_algoLength[] = { 4, 8, 16 };
static const char* XSUM_algoName[] = { "XXH32", "XXH64", "XXH128", "XXH3" };
static const char* XSUM_algoLE_name[] = { "XXH32_LE", "XXH64_LE", "XXH128_LE", "XXH3_LE" };
static const size_t XSUM_algoLength[] = { 4, 8, 16, 8 };
#define XSUM_TABLE_ELT_SIZE(table) (sizeof(table) / sizeof(*table))
@ -589,10 +296,16 @@ static void XSUM_printLine_BSD_internal(const char* filename,
const char* algoString[],
XSUM_displayHash_f f_displayHash)
{
assert(0 <= hashType && hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
{ const char* const typeString = algoString[hashType];
const size_t hashLength = XSUM_algoLength[hashType];
XSUM_output("%s (%s) = ", typeString, filename);
const int needsEscape = XSUM_filenameNeedsEscape(filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
XSUM_output("%s (", typeString);
XSUM_printFilename(filename, needsEscape);
XSUM_output(") = ");
f_displayHash(canonicalHash, hashLength);
XSUM_output("\n");
} }
@ -611,10 +324,16 @@ static void XSUM_printLine_GNU_internal(const char* filename,
const void* canonicalHash, const AlgoSelected hashType,
XSUM_displayHash_f f_displayHash)
{
assert(0 <= hashType && hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
{ const size_t hashLength = XSUM_algoLength[hashType];
const int needsEscape = XSUM_filenameNeedsEscape(filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
f_displayHash(canonicalHash, hashLength);
XSUM_output(" %s\n", filename);
XSUM_output(" ");
XSUM_printFilename(filename, needsEscape);
XSUM_output("\n");
} }
static void XSUM_printLine_GNU(const char* filename,
@ -688,22 +407,28 @@ static int XSUM_hashFile(const char* fileName,
{
case algo_xxh32:
{ XXH32_canonical_t hcbe32;
(void)XXH32_canonicalFromHash(&hcbe32, hashValue.xxh32);
(void)XXH32_canonicalFromHash(&hcbe32, hashValue.hash32);
f_displayLine(fileName, &hcbe32, hashType);
break;
}
case algo_xxh64:
{ XXH64_canonical_t hcbe64;
(void)XXH64_canonicalFromHash(&hcbe64, hashValue.xxh64);
(void)XXH64_canonicalFromHash(&hcbe64, hashValue.hash64);
f_displayLine(fileName, &hcbe64, hashType);
break;
}
case algo_xxh128:
{ XXH128_canonical_t hcbe128;
(void)XXH128_canonicalFromHash(&hcbe128, hashValue.xxh128);
(void)XXH128_canonicalFromHash(&hcbe128, hashValue.hash128);
f_displayLine(fileName, &hcbe128, hashType);
break;
}
case algo_xxh3:
{ XXH64_canonical_t hcbe64;
(void)XXH64_canonicalFromHash(&hcbe64, hashValue.hash64);
f_displayLine(fileName, &hcbe64, hashType);
break;
}
default:
assert(0); /* not possible */
}
@ -716,7 +441,7 @@ static int XSUM_hashFile(const char* fileName,
* XSUM_hashFiles:
* If fnTotal==0, read from stdin instead.
*/
static int XSUM_hashFiles(char*const * fnList, int fnTotal,
static int XSUM_hashFiles(const char* fnList[], int fnTotal,
AlgoSelected hashType,
Display_endianess displayEndianess,
Display_convention convention)
@ -764,9 +489,9 @@ typedef union {
} Canonical;
typedef struct {
Canonical canonical;
const char* filename;
int xxhBits; /* canonical type: 32:xxh32, 64:xxh64, 128:xxh128 */
Canonical canonical;
const char* filename;
AlgoSelected algo;
} ParsedLine;
typedef struct {
@ -785,10 +510,10 @@ typedef struct {
char* lineBuf;
size_t blockSize;
char* blockBuf;
XSUM_U32 strictMode;
XSUM_U32 statusOnly;
XSUM_U32 warn;
XSUM_U32 quiet;
XSUM_U32 strictMode;
XSUM_U32 statusOnly;
XSUM_U32 warn;
XSUM_U32 quiet;
ParseFileReport report;
} ParseFileArg;
@ -915,15 +640,14 @@ static CanonicalFromStringResult XSUM_canonicalFromString(unsigned char* dst,
*
* <algorithm> <' ('> <filename> <') = '> <hexstring> <'\0'>
*/
static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int rev)
static ParseLineResult XSUM_parseLine1(ParsedLine* parsedLine, char* line, int rev, int needsUnescape)
{
char* const firstSpace = strchr(line, ' ');
const char* hash_ptr;
size_t hash_len;
parsedLine->filename = NULL;
parsedLine->xxhBits = 0;
parsedLine->algo = algo_xxh64; /* default - will be overwritten */
if (firstSpace == NULL || !firstSpace[1]) return ParseLine_invalidFormat;
if (firstSpace[1] == '(') {
@ -935,43 +659,47 @@ static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int re
rev = strstr(line, "_LE") != NULL; /* was output little-endian */
hash_ptr = lastSpace + 1;
hash_len = strlen(hash_ptr);
/* NOTE: This currently ignores the hash description at the start of the string.
* In the future we should parse it and verify that it matches the hash length.
* It could also be used to allow both XXH64 & XXH3_64bits to be differentiated. */
if (!memcmp(line, "XXH3", 4)) parsedLine->algo = algo_xxh3;
if (!memcmp(line, "XXH32", 5)) parsedLine->algo = algo_xxh32;
if (!memcmp(line, "XXH64", 5)) parsedLine->algo = algo_xxh64;
if (!memcmp(line, "XXH128", 6)) parsedLine->algo = algo_xxh128;
} else {
hash_ptr = line;
hash_len = (size_t)(firstSpace - line);
if (hash_len==8) parsedLine->algo = algo_xxh32;
if (hash_len==16) parsedLine->algo = algo_xxh64;
if (hash_len==32) parsedLine->algo = algo_xxh128;
}
switch (hash_len)
{
case 8:
if (parsedLine->algo != algo_xxh32) return ParseLine_invalidFormat;
{ XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32;
if (XSUM_canonicalFromString(xxh32c->digest, sizeof(xxh32c->digest), hash_ptr, rev)
!= CanonicalFromString_ok) {
return ParseLine_invalidFormat;
}
parsedLine->xxhBits = 32;
break;
}
case 16:
if (parsedLine->algo != algo_xxh64 && parsedLine->algo != algo_xxh3) return ParseLine_invalidFormat;
{ XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64;
if (XSUM_canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), hash_ptr, rev)
!= CanonicalFromString_ok) {
return ParseLine_invalidFormat;
}
parsedLine->xxhBits = 64;
break;
}
case 32:
if (parsedLine->algo != algo_xxh128) return ParseLine_invalidFormat;
{ XXH128_canonical_t* xxh128c = &parsedLine->canonical.xxh128;
if (XSUM_canonicalFromString(xxh128c->digest, sizeof(xxh128c->digest), hash_ptr, rev)
!= CanonicalFromString_ok) {
return ParseLine_invalidFormat;
}
parsedLine->xxhBits = 128;
break;
}
@ -982,10 +710,29 @@ static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int re
/* note : skipping second separation character, which can be anything,
* allowing insertion of custom markers such as '*' */
parsedLine->filename = firstSpace + 2;
{
char* const filename = firstSpace + 2;
const size_t filenameLen = strlen(filename);
if (needsUnescape) {
char* const result = XSUM_filenameUnescape(filename, filenameLen);
if (result == NULL) {
return ParseLine_invalidFormat;
}
}
parsedLine->filename = filename;
}
return ParseLine_ok;
}
static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int rev) {
const int needsUnescape = XSUM_lineNeedsUnescape(line);
if (needsUnescape) {
++line;
}
return XSUM_parseLine1(parsedLine, line, rev, needsUnescape);
}
/*!
* Parse xxHash checksum file.
@ -1064,31 +811,31 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
break;
}
lineStatus = LineStatus_hashFailed;
switch (parsedLine.xxhBits)
{
case 32:
{ Multihash const xxh = XSUM_hashStream(fp, algo_xxh32, XSUM_parseFileArg->blockBuf, XSUM_parseFileArg->blockSize);
if (xxh.xxh32 == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) {
{ Multihash const xxh = XSUM_hashStream(fp, parsedLine.algo, XSUM_parseFileArg->blockBuf, XSUM_parseFileArg->blockSize);
switch (parsedLine.algo)
{
case algo_xxh32:
if (xxh.hash32 == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) {
lineStatus = LineStatus_hashOk;
} }
break;
}
break;
case 64:
{ Multihash const xxh = XSUM_hashStream(fp, algo_xxh64, XSUM_parseFileArg->blockBuf, XSUM_parseFileArg->blockSize);
if (xxh.xxh64 == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) {
case algo_xxh64:
case algo_xxh3:
if (xxh.hash64 == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) {
lineStatus = LineStatus_hashOk;
} }
break;
}
break;
case 128:
{ Multihash const xxh = XSUM_hashStream(fp, algo_xxh128, XSUM_parseFileArg->blockBuf, XSUM_parseFileArg->blockSize);
if (XXH128_isEqual(xxh.xxh128, XXH128_hashFromCanonical(&parsedLine.canonical.xxh128))) {
case algo_xxh128:
if (XXH128_isEqual(xxh.hash128, XXH128_hashFromCanonical(&parsedLine.canonical.xxh128))) {
lineStatus = LineStatus_hashOk;
} }
break;
}
break;
default:
break;
default:
break;
}
}
if (fp != stdin) fclose(fp);
} while (0);
@ -1119,8 +866,12 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
}
if (b && !XSUM_parseFileArg->statusOnly) {
XSUM_output("%s: %s\n", parsedLine.filename
, lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
const int needsEscape = XSUM_filenameNeedsEscape(parsedLine.filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
XSUM_printFilename(parsedLine.filename, needsEscape);
XSUM_output(": %s\n", lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
} }
break;
}
@ -1228,7 +979,7 @@ static int XSUM_checkFile(const char* inFileName,
}
static int XSUM_checkFiles(char*const* fnList, int fnTotal,
static int XSUM_checkFiles(const char* fnList[], int fnTotal,
const Display_endianess displayEndianess,
XSUM_U32 strictMode,
XSUM_U32 statusOnly,
@ -1261,7 +1012,7 @@ static int XSUM_usage(const char* exename)
XSUM_log( "Usage: %s [options] [files] \n\n", exename);
XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n");
XSUM_log( "Options: \n");
XSUM_log( " -H# algorithm selection: 0,1,2 or 32,64,128 (default: %i) \n", (int)g_defaultAlgo);
XSUM_log( " -H# algorithm selection: 0,1,2,3 or 32,64,128 (default: %i) \n", (int)g_defaultAlgo);
XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n");
XSUM_log( " -h, --help display a long help page about advanced options \n");
return 0;
@ -1277,7 +1028,7 @@ static int XSUM_usage_advanced(const char* exename)
XSUM_log( " --little-endian Checksum values use little endian convention (default: big endian) \n");
XSUM_log( " -b Run benchmark \n");
XSUM_log( " -b# Bench only algorithm variant # \n");
XSUM_log( " -i# Number of times to run the benchmark (default: %u) \n", (unsigned)g_nbIterations);
XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT);
XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n");
XSUM_log( "\n");
XSUM_log( "The following four options are useful only when verifying checksums (-c): \n");
@ -1358,7 +1109,7 @@ static XSUM_U32 XSUM_readU32FromChar(const char** stringPtr) {
return result;
}
XSUM_API int XSUM_main(int argc, char* argv[])
XSUM_API int XSUM_main(int argc, const char* argv[])
{
int i, filenamesStart = 0;
const char* const exename = XSUM_lastNameFromPath(argv[0]);
@ -1374,6 +1125,7 @@ XSUM_API int XSUM_main(int argc, char* argv[])
AlgoSelected algo = g_defaultAlgo;
Display_endianess displayEndianess = big_endian;
Display_convention convention = display_gnu;
int nbIterations = NBLOOPS_DEFAULT;
/* special case: xxhNNsum default to NN bits checksum */
if (strstr(exename, "xxh32sum") != NULL) algo = g_defaultAlgo = algo_xxh32;
@ -1414,7 +1166,9 @@ XSUM_API int XSUM_main(int argc, char* argv[])
{
/* Display version */
case 'V':
XSUM_log(FULL_WELCOME_MESSAGE(exename)); return 0;
XSUM_log(FULL_WELCOME_MESSAGE(exename));
XSUM_sanityCheck();
return 0;
/* Display help on XSUM_usage */
case 'h':
@ -1429,6 +1183,10 @@ XSUM_API int XSUM_main(int argc, char* argv[])
case 64: algo = algo_xxh64; break;
case 2 :
case 128: algo = algo_xxh128; break;
case 3 : /* xxh3 - necessarily uses BSD convention to avoid confusion with XXH64 */
algo = algo_xxh3;
convention = display_bsd;
break;
default:
return XSUM_badusage(exename);
}
@ -1453,17 +1211,18 @@ XSUM_API int XSUM_main(int argc, char* argv[])
do {
if (*argument == ',') argument++;
selectBenchIDs = XSUM_readU32FromChar(&argument); /* select one specific test */
if (selectBenchIDs < NB_TESTFUNC) {
if ((int)selectBenchIDs < g_nbTestFunctions) {
g_testIDs[selectBenchIDs] = 1;
} else
} else {
selectBenchIDs = kBenchAll;
}
} while (*argument == ',');
break;
/* Modify Nb Iterations (benchmark only) */
case 'i':
argument++;
g_nbIterations = XSUM_readU32FromChar(&argument);
nbIterations = (int)XSUM_readU32FromChar(&argument);
break;
/* Modify Block size (benchmark only) */
@ -1488,8 +1247,9 @@ XSUM_API int XSUM_main(int argc, char* argv[])
if (benchmarkMode) {
XSUM_logVerbose(2, FULL_WELCOME_MESSAGE(exename) );
XSUM_sanityCheck();
if (selectBenchIDs == 0) memcpy(g_testIDs, k_testIDs_default, sizeof(g_testIDs));
if (selectBenchIDs == kBenchAll) memset(g_testIDs, 1, sizeof(g_testIDs));
g_nbIterations = nbIterations;
if (selectBenchIDs == 0) memcpy(g_testIDs, k_testIDs_default, (size_t)g_nbTestFunctions);
if (selectBenchIDs == kBenchAll) memset(g_testIDs, 1, (size_t)g_nbTestFunctions);
if (filenamesStart==0) return XSUM_benchInternal(keySize);
return XSUM_benchFiles(argv+filenamesStart, argc-filenamesStart);
}

12
deps/xxHash/cmake_unofficial/CMakeLists.txt vendored
View file

@ -43,6 +43,15 @@ if(NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "xxHash build type: ${CMAKE_BUILD_TYPE}")
endif()
# Enable assert() statements in debug builds
if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
if("${CMAKE_VERSION}" VERSION_LESS "3.12")
# add_compile_definitions is not available for older cmake => do nothing
else()
add_compile_definitions(XXH_DEBUGLEVEL=1)
endif()
endif()
option(BUILD_SHARED_LIBS "Build shared library" ON)
option(XXHASH_BUILD_XXHSUM "Build the xxhsum binary" ON)
@ -88,6 +97,7 @@ if(XXHASH_BUILD_XXHSUM)
"${XXHSUM_DIR}/xsum_os_specific.c"
"${XXHSUM_DIR}/xsum_output.c"
"${XXHSUM_DIR}/xsum_sanity_check.c"
"${XXHSUM_DIR}/xsum_bench.c"
)
add_executable(${PROJECT_NAME}::xxhsum ALIAS xxhsum)
@ -117,7 +127,7 @@ if(NOT XXHASH_BUNDLED_MODE)
install(TARGETS xxhsum
EXPORT xxHashTargets
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(FILES "${XXHASH_DIR}/xxhsum.1"
install(FILES "${XXHSUM_DIR}/xxhsum.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
endif(XXHASH_BUILD_XXHSUM)

2
deps/xxHash/doc/xxhash_spec.md vendored
View file

@ -198,7 +198,7 @@ The algorithm uses 64-bit addition, multiplication, rotate, shift and xor operat
These constants are prime numbers, and feature a good mix of bits 1 and 0, neither too regular, nor too dissymmetric. These properties help dispersion capabilities.
### Step 1. Initialise internal accumulators
### Step 1. Initialize internal accumulators
Each accumulator gets an initial value based on optional `seed` input. Since the `seed` is optional, it can be `0`.

2
deps/xxHash/libxxhash.pc.in vendored
View file

@ -1,5 +1,5 @@
# xxHash - Extremely fast hash algorithm
# Copyright (C) 2012-2020, Yann Collet, Facebook
# Copyright (C) 2012-2021, Yann Collet, Facebook
# BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@

6
deps/xxHash/tests/Makefile vendored
View file

@ -1,6 +1,6 @@
# ################################################################
# xxHash Makefile
# Copyright (C) 2012-2020 Yann Collet
# Copyright (C) 2012-2021 Yann Collet
#
# GPL v2 License
#
@ -100,6 +100,10 @@ test_unicode: $(XXHSUM) generate_unicode_test.c
$(SHELL) ./unicode_test.sh
endif
.PHONY: test_filename_escape
test_filename_escape: $(XXHSUM)
./filename-escape.sh
xxhash.o: ../xxhash.c ../xxhash.h
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -c -o $@ $<

2
deps/xxHash/tests/bench/Makefile vendored
View file

@ -1,6 +1,6 @@
# ################################################################
# xxHash benchHash Makefile
# Copyright (C) 2019-2020 Yann Collet
# Copyright (C) 2019-2021 Yann Collet
#
# GPL v2 License
#

2
deps/xxHash/tests/bench/benchHash.c vendored
View file

@ -1,7 +1,7 @@
/*
* Hash benchmark module
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

2
deps/xxHash/tests/bench/benchHash.h vendored
View file

@ -1,7 +1,7 @@
/*
* Hash benchmark module
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

2
deps/xxHash/tests/bench/benchfn.c vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/bench/benchfn.h vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/bench/bhDisplay.c vendored
View file

@ -1,7 +1,7 @@
/*
* CSV Display module for the hash benchmark program
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

2
deps/xxHash/tests/bench/bhDisplay.h vendored
View file

@ -1,7 +1,7 @@
/*
* CSV Display module for the hash benchmark program
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

2
deps/xxHash/tests/bench/hashes.h vendored
View file

@ -1,7 +1,7 @@
/*
* List hash algorithms to benchmark
* Part of xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

2
deps/xxHash/tests/bench/main.c vendored
View file

@ -1,7 +1,7 @@
/*
* Main program to benchmark hash functions
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

2
deps/xxHash/tests/bench/timefn.c vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2019-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/bench/timefn.h vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/collisions/Makefile vendored
View file

@ -1,6 +1,6 @@
# Brute force collision tester for 64-bit hashes
# Part of xxHash project
# Copyright (C) 2019-2020 Yann Collet
# Copyright (C) 2019-2021 Yann Collet
#
# GPL v2 License
#

2
deps/xxHash/tests/collisions/hashes.h vendored
View file

@ -1,7 +1,7 @@
/*
* List of hashes for the brute force collision tester
* Part of xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

4
deps/xxHash/tests/collisions/main.c vendored
View file

@ -1,7 +1,7 @@
/*
* Brute force collision tester for 64-bit hashes
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*
@ -800,7 +800,7 @@ static size_t search_collisions(
for (int nbHBits = 1; nbHBits < hashBits; nbHBits++) {
uint64_t const nbSlots = (uint64_t)1 << nbHBits;
double const expectedCollisions = estimateNbCollisions(nbCandidates, nbHBits);
if ( (nbSlots > nbCandidates * 100) /* within range for meaningfull collision analysis results */
if ( (nbSlots > nbCandidates * 100) /* within range for meaningful collision analysis results */
&& (expectedCollisions > 18.0) ) {
int const rShift = hashBits - nbHBits;
size_t HBits_collisions = 0;

2
deps/xxHash/tests/collisions/pool.c vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/collisions/pool.h vendored
View file

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

2
deps/xxHash/tests/collisions/sort.cc vendored
View file

@ -1,6 +1,6 @@
/*
* sort.cc - C++ sort functions
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

2
deps/xxHash/tests/collisions/sort.hh vendored
View file

@ -1,6 +1,6 @@
/*
* sort.hh - headers for C++ sort functions
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

2
deps/xxHash/xxh3.h vendored
View file

@ -1,7 +1,7 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Development source file for `xxh3`
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*

8
deps/xxHash/xxh_x86dispatch.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2020 Yann Collet
* Copyright (C) 2020-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@ -36,7 +36,7 @@
/*!
* @file xxh_x86dispatch.c
*
* Automatic dispatcher code for the @ref xxh3_family on x86-based targets.
* Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
*
* Optional add-on.
*
@ -229,7 +229,7 @@ extern "C" {
* @internal
* @brief Runs CPUID.
*
* @param eax, ecx The parameters to pass to CPUID, %eax and %ecx respectively.
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
* @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }`
*/
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
@ -316,7 +316,7 @@ static xxh_u64 XXH_xgetbv(void)
*
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
*
* @ret The best @ref XXH_VECTOR implementation.
* @return The best @ref XXH_VECTOR implementation.
* @see XXH_VECTOR_TYPES
*/
static int XXH_featureTest(void)

3
deps/xxHash/xxh_x86dispatch.h vendored
View file

@ -1,6 +1,6 @@
/*
* xxHash - XXH3 Dispatcher for x86-based targets
* Copyright (C) 2020 Yann Collet
* Copyright (C) 2020-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@ -71,7 +71,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, c
# undef XXH128
# define XXH128 XXH3_128bits_withSeed_dispatch
# define XXH3_128bits XXH3_128bits_dispatch
# undef XXH3_128bits
# define XXH3_128bits XXH3_128bits_dispatch
# undef XXH3_128bits_withSeed

2
deps/xxHash/xxhash.c vendored
View file

@ -1,6 +1,6 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2012-2020 Yann Collet
* Copyright (C) 2012-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*

2006
deps/xxHash/xxhash.h vendored
View file

File diff suppressed because it is too large Load diff