summaryrefslogtreecommitdiff
path: root/Externals
diff options
context:
space:
mode:
authorTao Schreiner <taoschreiner@outlook.com>2017-06-10 14:02:27 +0200
committerTao Schreiner <taoschreiner@outlook.com>2017-06-10 14:02:27 +0200
commitab2fca042e0d8bfbc98e4e9ac69732550cfb6aed (patch)
tree23bdf3f8fae64db55e65b5e169ed736aefe258d1 /Externals
parent0a6933137109eb26b2efe4795a8a0ee30a9ecdc5 (diff)
Update xxhash to 0.6.2
Diffstat (limited to 'Externals')
-rw-r--r--Externals/xxhash/xxhash.c1168
-rw-r--r--Externals/xxhash/xxhash.h265
-rw-r--r--Externals/xxhash/xxhsum.c1387
3 files changed, 1738 insertions, 1082 deletions
diff --git a/Externals/xxhash/xxhash.c b/Externals/xxhash/xxhash.c
index 24a64b5f80..4546c54dbe 100644
--- a/Externals/xxhash/xxhash.c
+++ b/Externals/xxhash/xxhash.c
@@ -1,154 +1,183 @@
/*
-xxHash - Fast Hash algorithm
-Copyright (C) 2012-2014, Yann Collet.
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash source repository : http://code.google.com/p/xxhash/
-- public discussion board : https://groups.google.com/forum/#!forum/lz4c
+* xxHash - Fast Hash algorithm
+* Copyright (C) 2012-2016, Yann Collet
+*
+* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are
+* met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above
+* copyright notice, this list of conditions and the following disclaimer
+* in the documentation and/or other materials provided with the
+* distribution.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+* You can contact the author at :
+* - xxHash homepage: http://www.xxhash.com
+* - xxHash source repository : https://github.com/Cyan4973/xxHash
*/
-//**************************************
-// Tuning parameters
-//**************************************
-// Unaligned memory access is automatically enabled for "common" CPU, such as x86.
-// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
-// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
-// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
-#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-# define XXH_USE_UNALIGNED_ACCESS 1
+/* *************************************
+* Tuning parameters
+***************************************/
+/*!XXH_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
+ * It can generate buggy code on targets which do not support unaligned memory accesses.
+ * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://stackoverflow.com/a/32095106/646947 for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
+# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+# define XXH_FORCE_MEMORY_ACCESS 2
+# elif defined(__INTEL_COMPILER) || \
+ (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+# define XXH_FORCE_MEMORY_ACCESS 1
+# endif
#endif
-// XXH_ACCEPT_NULL_INPUT_POINTER :
-// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
-// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
-// This option has a very small performance cost (only measurable on small inputs).
-// By default, this option is disabled. To enable it, uncomment below define :
-// #define XXH_ACCEPT_NULL_INPUT_POINTER 1
-
-// XXH_FORCE_NATIVE_FORMAT :
-// By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
-// Results are therefore identical for little-endian and big-endian CPU.
-// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
-// Should endian-independance be of no importance for your application, you may set the #define below to 1.
-// It will improve speed for Big-endian CPU.
-// This option has no impact on Little_Endian CPU.
-#define XXH_FORCE_NATIVE_FORMAT 0
-
-//**************************************
-// Compiler Specific Options
-//**************************************
-// Disable some Visual warning messages
-#ifdef _MSC_VER // Visual Studio
-# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant
+/*!XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/*!XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independance be of no importance for your application, you may set the #define below to 1,
+ * to improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
+# define XXH_FORCE_NATIVE_FORMAT 0
#endif
-#ifdef _MSC_VER // Visual Studio
-# define FORCE_INLINE static __forceinline
-#else
-# ifdef __GNUC__
-# define FORCE_INLINE static inline __attribute__((always_inline))
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash; set to 0 when the input data
+ * is guaranteed to be aligned.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+# define XXH_FORCE_ALIGN_CHECK 0
# else
-# define FORCE_INLINE static inline
+# define XXH_FORCE_ALIGN_CHECK 1
# endif
#endif
-//**************************************
-// Includes & Memory related functions
-//**************************************
-#include "xxhash.h"
-// Modify the local functions below should you wish to use some other memory routines
-// for malloc(), free()
+
+/* *************************************
+* Includes & Memory related functions
+***************************************/
+/* Modify the local functions below should you wish to use some other memory routines */
+/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
-// for memcpy()
+/* for memcpy() */
#include <string.h>
-static void* XXH_memcpy(void* dest, const void* src, size_t size)
-{
- return memcpy(dest,src,size);
-}
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
-//**************************************
-// Basic Types
-//**************************************
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
-typedef uint8_t BYTE;
-typedef uint16_t U16;
-typedef uint32_t U32;
-typedef int32_t S32;
-typedef uint64_t U64;
-#else
-typedef unsigned char BYTE;
-typedef unsigned short U16;
-typedef unsigned int U32;
-typedef signed int S32;
-typedef unsigned long long U64;
-#endif
-#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS)
-# define _PACKED __attribute__ ((packed))
+/* *************************************
+* Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER /* Visual Studio */
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
+# define FORCE_INLINE static __forceinline
#else
-# define _PACKED
-#endif
-
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# ifdef __IBMC__
-# pragma pack(1)
+# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# ifdef __GNUC__
+# define FORCE_INLINE static inline __attribute__((always_inline))
+# else
+# define FORCE_INLINE static inline
+# endif
# else
-# pragma pack(push, 1)
-# endif
+# define FORCE_INLINE static
+# endif /* __STDC_VERSION__ */
#endif
-typedef struct _U32_S
-{
- U32 v;
-} _PACKED U32_S;
-typedef struct _U64_S
-{
- U64 v;
-} _PACKED U64_S;
-#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__)
-# pragma pack(pop)
+/* *************************************
+* Basic Types
+***************************************/
+#ifndef MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+# else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+# endif
#endif
-#define A32(x) (((U32_S *)(x))->v)
-#define A64(x) (((U64_S *)(x))->v)
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; } __attribute__((packed)) unalign;
+static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+
+#else
-//***************************************
-// Compiler-specific Functions and Macros
-//***************************************
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+static U32 XXH_read32(const void* memPtr)
+{
+ U32 val;
+ memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+
+/* ****************************************
+* Compiler-specific Functions and Macros
+******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
-// Note : although _rotl exists for minGW (GCC under windows), performance seems poor
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
#if defined(_MSC_VER)
# define XXH_rotl32(x,r) _rotl(x,r)
# define XXH_rotl64(x,r) _rotl64(x,r)
@@ -157,76 +186,44 @@ typedef struct _U64_S
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
#endif
-#if defined(_MSC_VER) // Visual Studio
+#if defined(_MSC_VER) /* Visual Studio */
# define XXH_swap32 _byteswap_ulong
-# define XXH_swap64 _byteswap_uint64
#elif GCC_VERSION >= 403
# define XXH_swap32 __builtin_bswap32
-# define XXH_swap64 __builtin_bswap64
#else
-static inline U32 XXH_swap32 (U32 x)
+static U32 XXH_swap32 (U32 x)
{
return ((x << 24) & 0xff000000 ) |
((x << 8) & 0x00ff0000 ) |
((x >> 8) & 0x0000ff00 ) |
((x >> 24) & 0x000000ff );
}
-static inline U64 XXH_swap64 (U64 x)
-{
- return ((x << 56) & 0xff00000000000000ULL) |
- ((x << 40) & 0x00ff000000000000ULL) |
- ((x << 24) & 0x0000ff0000000000ULL) |
- ((x << 8) & 0x000000ff00000000ULL) |
- ((x >> 8) & 0x00000000ff000000ULL) |
- ((x >> 24) & 0x0000000000ff0000ULL) |
- ((x >> 40) & 0x000000000000ff00ULL) |
- ((x >> 56) & 0x00000000000000ffULL);
-}
#endif
-//**************************************
-// Constants
-//**************************************
-#define PRIME32_1 2654435761U
-#define PRIME32_2 2246822519U
-#define PRIME32_3 3266489917U
-#define PRIME32_4 668265263U
-#define PRIME32_5 374761393U
-
-#define PRIME64_1 11400714785074694791ULL
-#define PRIME64_2 14029467366897019727ULL
-#define PRIME64_3 1609587929392839161ULL
-#define PRIME64_4 9650029242287828579ULL
-#define PRIME64_5 2870177450012600261ULL
-
-//**************************************
-// Architecture Macros
-//**************************************
+/* *************************************
+* Architecture Macros
+***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
-#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch
-static const int one = 1;
-# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one))
-#endif
-
-//**************************************
-// Macros
-//**************************************
-#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations
+/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
+#ifndef XXH_CPU_LITTLE_ENDIAN
+ static const int g_one = 1;
+# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
+#endif
-//****************************
-// Memory reads
-//****************************
+/* ***************************
+* Memory reads
+*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
- return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr));
+ return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
else
- return endian==XXH_littleEndian ? *(U32*)ptr : XXH_swap32(*(U32*)ptr);
+ return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
@@ -234,23 +231,36 @@ FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
-FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+static U32 XXH_readBE32(const void* ptr)
{
- if (align==XXH_unaligned)
- return endian==XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr));
- else
- return endian==XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr);
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
}
-FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+
+/* *************************************
+* Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
+
+
+/* *******************************************************************
+* 32-bits hash functions
+*********************************************************************/
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 = 668265263U;
+static const U32 PRIME32_5 = 374761393U;
+
+static U32 XXH32_round(U32 seed, U32 input)
{
- return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+ seed += input * PRIME32_2;
+ seed = XXH_rotl32(seed, 13);
+ seed *= PRIME32_1;
+ return seed;
}
-
-//****************************
-// Simple Hash Functions
-//****************************
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
@@ -259,60 +269,40 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL)
- {
+ if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)16;
}
#endif
- if (len>=16)
- {
+ if (len>=16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1;
- do
- {
- v1 += XXH_get32bits(p) * PRIME32_2;
- v1 = XXH_rotl32(v1, 13);
- v1 *= PRIME32_1;
- p+=4;
- v2 += XXH_get32bits(p) * PRIME32_2;
- v2 = XXH_rotl32(v2, 13);
- v2 *= PRIME32_1;
- p+=4;
- v3 += XXH_get32bits(p) * PRIME32_2;
- v3 = XXH_rotl32(v3, 13);
- v3 *= PRIME32_1;
- p+=4;
- v4 += XXH_get32bits(p) * PRIME32_2;
- v4 = XXH_rotl32(v4, 13);
- v4 *= PRIME32_1;
- p+=4;
- }
- while (p<=limit);
+ do {
+ v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+ v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+ v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+ v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+ } while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
- }
- else
- {
+ } else {
h32 = seed + PRIME32_5;
}
h32 += (U32) len;
- while (p+4<=bEnd)
- {
+ while (p+4<=bEnd) {
h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++;
@@ -328,26 +318,24 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH
}
-unsigned int XXH32 (const void* input, size_t len, unsigned seed)
+XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
{
#if 0
- // Simple version, good for code maintenance, but unfortunately slow for small inputs
- XXH32_state_t state;
- XXH32_reset(&state, seed);
- XXH32_update(&state, input, len);
- return XXH32_digest(&state);
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH32_CREATESTATE_STATIC(state);
+ XXH32_reset(state, seed);
+ XXH32_update(state, input, len);
+ return XXH32_digest(state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-# if !defined(XXH_USE_UNALIGNED_ACCESS)
- if ((((size_t)input) & 3) == 0) // Input is aligned, let's leverage the speed advantage
- {
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- }
-# endif
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
@@ -356,231 +344,40 @@ unsigned int XXH32 (const void* input, size_t len, unsigned seed)
#endif
}
-FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
-{
- const BYTE* p = (const BYTE*)input;
- const BYTE* bEnd = p + len;
- U64 h64;
-#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
-#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL)
- {
- len=0;
- bEnd=p=(const BYTE*)(size_t)32;
- }
-#endif
- if (len>=32)
- {
- const BYTE* const limit = bEnd - 32;
- U64 v1 = seed + PRIME64_1 + PRIME64_2;
- U64 v2 = seed + PRIME64_2;
- U64 v3 = seed + 0;
- U64 v4 = seed - PRIME64_1;
+/*====== Hash streaming ======*/
- do
- {
- v1 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- v2 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- v3 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- v4 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- }
- while (p<=limit);
-
- h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-
- v1 *= PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- h64 ^= v1;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v2 *= PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- h64 ^= v2;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v3 *= PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- h64 ^= v3;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v4 *= PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- h64 ^= v4;
- h64 = h64 * PRIME64_1 + PRIME64_4;
- }
- else
- {
- h64 = seed + PRIME64_5;
- }
-
- h64 += (U64) len;
-
- while (p+8<=bEnd)
- {
- U64 k1 = XXH_get64bits(p);
- k1 *= PRIME64_2;
- k1 = XXH_rotl64(k1,31);
- k1 *= PRIME64_1;
- h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
- p+=8;
- }
-
- if (p+4<=bEnd)
- {
- h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
- p+=4;
- }
-
- while (p<bEnd)
- {
- h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
- p++;
- }
-
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
-
- return h64;
-}
-
-
-unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{
-#if 0
- // Simple version, good for code maintenance, but unfortunately slow for small inputs
- XXH64_state_t state;
- XXH64_reset(&state, seed);
- XXH64_update(&state, input, len);
- return XXH64_digest(&state);
-#else
- XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-
-# if !defined(XXH_USE_UNALIGNED_ACCESS)
- if ((((size_t)input) & 7)==0) // Input is aligned, let's leverage the speed advantage
- {
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- }
-# endif
-
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
- else
- return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
-#endif
-}
-
-/****************************************************
- * Advanced Hash Functions
-****************************************************/
-
-/*** Allocation ***/
-typedef struct
-{
- U64 total_len;
- U32 seed;
- U32 v1;
- U32 v2;
- U32 v3;
- U32 v4;
- U32 mem32[4]; /* defined as U32 for alignment */
- U32 memsize;
-} XXH_istate32_t;
-
-typedef struct
-{
- U64 total_len;
- U64 seed;
- U64 v1;
- U64 v2;
- U64 v3;
- U64 v4;
- U64 mem64[4]; /* defined as U64 for alignment */
- U32 memsize;
-} XXH_istate64_t;
-
-
-XXH32_state_t* XXH32_createState(void)
-{
- XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); // A compilation error here means XXH32_state_t is not large enough
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
-XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
-};
-
-XXH64_state_t* XXH64_createState(void)
-{
- XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); // A compilation error here means XXH64_state_t is not large enough
- return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
-XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
-{
- XXH_free(statePtr);
- return XXH_OK;
-};
-
-/*** Hash feed ***/
-
-XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed)
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
{
- XXH_istate32_t* state = (XXH_istate32_t*) state_in;
- state->seed = seed;
- state->v1 = seed + PRIME32_1 + PRIME32_2;
- state->v2 = seed + PRIME32_2;
- state->v3 = seed + 0;
- state->v4 = seed - PRIME32_1;
- state->total_len = 0;
- state->memsize = 0;
- return XXH_OK;
+ memcpy(dstState, srcState, sizeof(*dstState));
}
-XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
{
- XXH_istate64_t* state = (XXH_istate64_t*) state_in;
- state->seed = seed;
- state->v1 = seed + PRIME64_1 + PRIME64_2;
- state->v2 = seed + PRIME64_2;
- state->v3 = seed + 0;
- state->v4 = seed - PRIME64_1;
- state->total_len = 0;
- state->memsize = 0;
+ XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */
+ state.v1 = seed + PRIME32_1 + PRIME32_2;
+ state.v2 = seed + PRIME32_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME32_1;
+ memcpy(statePtr, &state, sizeof(state));
return XXH_OK;
}
-FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
- XXH_istate32_t* state = (XXH_istate32_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@@ -588,69 +385,40 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v
if (input==NULL) return XXH_ERROR;
#endif
- state->total_len += len;
+ state->total_len_32 += (unsigned)len;
+ state->large_len |= (len>=16) | (state->total_len_32>=16);
- if (state->memsize + len < 16) // fill in tmp buffer
- {
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
- state->memsize += (U32)len;
+ state->memsize += (unsigned)len;
return XXH_OK;
}
- if (state->memsize) // some data left from previous update
- {
+ if (state->memsize) { /* some data left from previous update */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
- {
- const U32* p32 = state->mem32;
- state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v1 = XXH_rotl32(state->v1, 13);
- state->v1 *= PRIME32_1;
- p32++;
- state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v2 = XXH_rotl32(state->v2, 13);
- state->v2 *= PRIME32_1;
- p32++;
- state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v3 = XXH_rotl32(state->v3, 13);
- state->v3 *= PRIME32_1;
- p32++;
- state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v4 = XXH_rotl32(state->v4, 13);
- state->v4 *= PRIME32_1;
- p32++;
+ { const U32* p32 = state->mem32;
+ state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+ state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+ state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+ state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
}
p += 16-state->memsize;
state->memsize = 0;
}
- if (p <= bEnd-16)
- {
+ if (p <= bEnd-16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1;
U32 v2 = state->v2;
U32 v3 = state->v3;
U32 v4 = state->v4;
- do
- {
- v1 += XXH_readLE32(p, endian) * PRIME32_2;
- v1 = XXH_rotl32(v1, 13);
- v1 *= PRIME32_1;
- p+=4;
- v2 += XXH_readLE32(p, endian) * PRIME32_2;
- v2 = XXH_rotl32(v2, 13);
- v2 *= PRIME32_1;
- p+=4;
- v3 += XXH_readLE32(p, endian) * PRIME32_2;
- v3 = XXH_rotl32(v3, 13);
- v3 *= PRIME32_1;
- p+=4;
- v4 += XXH_readLE32(p, endian) * PRIME32_2;
- v4 = XXH_rotl32(v4, 13);
- v4 *= PRIME32_1;
- p+=4;
- }
- while (p<=limit);
+ do {
+ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+ v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+ v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+ v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+ } while (p<=limit);
state->v1 = v1;
state->v2 = v2;
@@ -658,16 +426,15 @@ FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const v
state->v4 = v4;
}
- if (p < bEnd)
- {
- XXH_memcpy(state->mem32, p, bEnd-p);
- state->memsize = (int)(bEnd-p);
+ if (p < bEnd) {
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
}
return XXH_OK;
}
-XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@@ -679,35 +446,29 @@ XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t l
-FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
- XXH_istate32_t* state = (XXH_istate32_t*) state_in;
const BYTE * p = (const BYTE*)state->mem32;
- BYTE* bEnd = (BYTE*)(state->mem32) + state->memsize;
+ const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
- if (state->total_len >= 16)
- {
+ if (state->large_len) {
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
- }
- else
- {
- h32 = state->seed + PRIME32_5;
+ } else {
+ h32 = state->v3 /* == seed */ + PRIME32_5;
}
- h32 += (U32) state->total_len;
+ h32 += state->total_len_32;
- while (p+4<=bEnd)
- {
+ while (p+4<=bEnd) {
h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++;
}
@@ -721,7 +482,7 @@ FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endiane
}
-U32 XXH32_digest (const XXH32_state_t* state_in)
+XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@@ -732,9 +493,261 @@ U32 XXH32_digest (const XXH32_state_t* state_in)
}
-FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+/*====== Canonical representation ======*/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+ return XXH_readBE32(src);
+}
+
+
+#ifndef XXH_NO_LONG_LONG
+
+/* *******************************************************************
+* 64-bits hash functions
+*********************************************************************/
+
+/*====== Memory access ======*/
+
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+ typedef uint64_t U64;
+# else
+ typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */
+# endif
+#endif
+
+
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
+
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
+static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64;
+
+static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; }
+
+#else
+
+/* portable and safe solution. Generally efficient.
+ * see : http://stackoverflow.com/a/32095106/646947
+ */
+
+static U64 XXH_read64(const void* memPtr)
+{
+ U64 val;
+ memcpy(&val, memPtr, sizeof(val));
+ return val;
+}
+
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
+
+#if defined(_MSC_VER) /* Visual Studio */
+# define XXH_swap64 _byteswap_uint64
+#elif GCC_VERSION >= 403
+# define XXH_swap64 __builtin_bswap64
+#else
+static U64 XXH_swap64 (U64 x)
+{
+ return ((x << 56) & 0xff00000000000000ULL) |
+ ((x << 40) & 0x00ff000000000000ULL) |
+ ((x << 24) & 0x0000ff0000000000ULL) |
+ ((x << 8) & 0x000000ff00000000ULL) |
+ ((x >> 8) & 0x00000000ff000000ULL) |
+ ((x >> 24) & 0x0000000000ff0000ULL) |
+ ((x >> 40) & 0x000000000000ff00ULL) |
+ ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+ if (align==XXH_unaligned)
+ return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+ else
+ return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+ return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+static U64 XXH_readBE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
+
+/*====== xxh64 ======*/
+
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 = 1609587929392839161ULL;
+static const U64 PRIME64_4 = 9650029242287828579ULL;
+static const U64 PRIME64_5 = 2870177450012600261ULL;
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+ acc += input * PRIME64_2;
+ acc = XXH_rotl64(acc, 31);
+ acc *= PRIME64_1;
+ return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+ val = XXH64_round(0, val);
+ acc ^= val;
+ acc = acc * PRIME64_1 + PRIME64_4;
+ return acc;
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+ const BYTE* p = (const BYTE*)input;
+ const BYTE* const bEnd = p + len;
+ U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+ if (p==NULL) {
+ len=0;
+ bEnd=p=(const BYTE*)(size_t)32;
+ }
+#endif
+
+ if (len>=32) {
+ const BYTE* const limit = bEnd - 32;
+ U64 v1 = seed + PRIME64_1 + PRIME64_2;
+ U64 v2 = seed + PRIME64_2;
+ U64 v3 = seed + 0;
+ U64 v4 = seed - PRIME64_1;
+
+ do {
+ v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+ v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+ v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+ v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+ } while (p<=limit);
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+
+ } else {
+ h64 = seed + PRIME64_5;
+ }
+
+ h64 += (U64) len;
+
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_get64bits(p));
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ p+=8;
+ }
+
+ if (p+4<=bEnd) {
+ h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ p+=4;
+ }
+
+ while (p<bEnd) {
+ h64 ^= (*p) * PRIME64_5;
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+
+XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+ XXH64_CREATESTATE_STATIC(state);
+ XXH64_reset(state, seed);
+ XXH64_update(state, input, len);
+ return XXH64_digest(state);
+#else
+ XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
+
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+ else
+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/*====== Hash Streaming ======*/
+
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
+{
+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+ XXH_free(statePtr);
+ return XXH_OK;
+}
+
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
+{
+ memcpy(dstState, srcState, sizeof(*dstState));
+}
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
+{
+ XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
+ memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */
+ state.v1 = seed + PRIME64_1 + PRIME64_2;
+ state.v2 = seed + PRIME64_2;
+ state.v3 = seed + 0;
+ state.v4 = seed - PRIME64_1;
+ memcpy(statePtr, &state, sizeof(state));
+ return XXH_OK;
+}
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
- XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
@@ -744,67 +757,35 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v
state->total_len += len;
- if (state->memsize + len < 32) // fill in tmp buffer
- {
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
- if (state->memsize) // some data left from previous update
- {
+ if (state->memsize) { /* tmp buffer is full */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
- {
- const U64* p64 = state->mem64;
- state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v1 = XXH_rotl64(state->v1, 31);
- state->v1 *= PRIME64_1;
- p64++;
- state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v2 = XXH_rotl64(state->v2, 31);
- state->v2 *= PRIME64_1;
- p64++;
- state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v3 = XXH_rotl64(state->v3, 31);
- state->v3 *= PRIME64_1;
- p64++;
- state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v4 = XXH_rotl64(state->v4, 31);
- state->v4 *= PRIME64_1;
- p64++;
- }
+ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+ state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+ state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+ state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
p += 32-state->memsize;
state->memsize = 0;
}
- if (p+32 <= bEnd)
- {
+ if (p+32 <= bEnd) {
const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
- do
- {
- v1 += XXH_readLE64(p, endian) * PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- p+=8;
- v2 += XXH_readLE64(p, endian) * PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- p+=8;
- v3 += XXH_readLE64(p, endian) * PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- p+=8;
- v4 += XXH_readLE64(p, endian) * PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- p+=8;
- }
- while (p<=limit);
+ do {
+ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+ v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+ v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+ v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+ } while (p<=limit);
state->v1 = v1;
state->v2 = v2;
@@ -812,16 +793,15 @@ FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const v
state->v4 = v4;
}
- if (p < bEnd)
- {
- XXH_memcpy(state->mem64, p, bEnd-p);
- state->memsize = (int)(bEnd-p);
+ if (p < bEnd) {
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
+ state->memsize = (unsigned)(bEnd-p);
}
return XXH_OK;
}
-XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@@ -831,77 +811,45 @@ XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t l
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
-
-
-FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
- XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const BYTE * p = (const BYTE*)state->mem64;
- BYTE* bEnd = (BYTE*)state->mem64 + state->memsize;
+ const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
- if (state->total_len >= 32)
- {
- U64 v1 = state->v1;
- U64 v2 = state->v2;
- U64 v3 = state->v3;
- U64 v4 = state->v4;
+ if (state->total_len >= 32) {
+ U64 const v1 = state->v1;
+ U64 const v2 = state->v2;
+ U64 const v3 = state->v3;
+ U64 const v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-
- v1 *= PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- h64 ^= v1;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v2 *= PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- h64 ^= v2;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v3 *= PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- h64 ^= v3;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v4 *= PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- h64 ^= v4;
- h64 = h64*PRIME64_1 + PRIME64_4;
- }
- else
- {
- h64 = state->seed + PRIME64_5;
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+ } else {
+ h64 = state->v3 + PRIME64_5;
}
h64 += (U64) state->total_len;
- while (p+8<=bEnd)
- {
- U64 k1 = XXH_readLE64(p, endian);
- k1 *= PRIME64_2;
- k1 = XXH_rotl64(k1,31);
- k1 *= PRIME64_1;
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
- if (p+4<=bEnd)
- {
+ if (p+4<=bEnd) {
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
@@ -914,8 +862,7 @@ FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endiane
return h64;
}
-
-unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
@@ -926,3 +873,18 @@ unsigned long long XXH64_digest (const XXH64_state_t* state_in)
}
+/*====== Canonical representation ======*/
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+ return XXH_readBE64(src);
+}
+
+#endif /* XXH_NO_LONG_LONG */
diff --git a/Externals/xxhash/xxhash.h b/Externals/xxhash/xxhash.h
index 55b45015a4..947567397b 100644
--- a/Externals/xxhash/xxhash.h
+++ b/Externals/xxhash/xxhash.h
@@ -1,7 +1,8 @@
/*
xxHash - Extremely Fast Hash algorithm
Header File
- Copyright (C) 2012-2014, Yann Collet.
+ Copyright (C) 2012-2016, Yann Collet.
+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
@@ -28,7 +29,7 @@
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- - xxHash source repository : http://code.google.com/p/xxhash/
+ - xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* Notice extracted from xxHash homepage :
@@ -55,102 +56,246 @@ SHA1-32 0.28 GB/s 10
Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name Speed on 64 bits Speed on 32 bits
+XXH64 13.8 GB/s 1.9 GB/s
+XXH32 6.8 GB/s 6.0 GB/s
*/
-#pragma once
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
#if defined (__cplusplus)
extern "C" {
#endif
-/*****************************
- Includes
-*****************************/
-#include <stddef.h> /* size_t */
+/* ****************************
+* Compiler specifics
+******************************/
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */
+# define restrict /* disable restrict */
+#endif
-/*****************************
- Type
-*****************************/
+/* ****************************
+* Definitions
+******************************/
+#include <stddef.h> /* size_t */
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+/* ****************************
+* API modifier
+******************************/
+/** XXH_PRIVATE_API
+* This is useful to include xxhash functions in `static` mode
+* in order to inline them, and remove their symbol from the public list.
+* Methodology :
+* #define XXH_PRIVATE_API
+* #include "xxhash.h"
+* `xxhash.c` is automatically included.
+* It's not useful to compile and link it as a separate module.
+*/
+#ifdef XXH_PRIVATE_API
+# ifndef XXH_STATIC_LINKING_ONLY
+# define XXH_STATIC_LINKING_ONLY
+# endif
+# if defined(__GNUC__)
+# define XXH_PUBLIC_API static __inline __attribute__((unused))
+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# define XXH_PUBLIC_API static inline
+# elif defined(_MSC_VER)
+# define XXH_PUBLIC_API static __inline
+# else
+# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
+# endif
+#else
+# define XXH_PUBLIC_API /* do nothing */
+#endif /* XXH_PRIVATE_API */
+
+/*!XXH_NAMESPACE, aka Namespace Emulation :
+
+If you want to include _and expose_ xxHash functions from within your own library,
+but also want to avoid symbol collisions with other libraries which may also include xxHash,
+
+you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
+with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values).
+
+Note that no change is required within the calling program as long as it includes `xxhash.h` :
+regular symbol name will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+# define XXH_CAT(A,B) A##B
+# define XXH_NAME2(A,B) XXH_CAT(A,B)
+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
+#endif
-/*****************************
- Simple Hash Functions
-*****************************/
-unsigned int XXH32 (const void* input, size_t length, unsigned seed);
-unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
+/* *************************************
+* Version
+***************************************/
+#define XXH_VERSION_MAJOR 0
+#define XXH_VERSION_MINOR 6
+#define XXH_VERSION_RELEASE 2
+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
+XXH_PUBLIC_API unsigned XXH_versionNumber (void);
-/*
-XXH32() :
+
+/*-**********************************************************************
+* 32-bits hash
+************************************************************************/
+typedef unsigned int XXH32_hash_t;
+
+/*! XXH32() :
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
- This function successfully passes all SMHasher tests.
- Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
-XXH64() :
- Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
-*/
-
+ Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+/*====== Streaming ======*/
+typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
-/*****************************
- Advanced Hash Functions
-*****************************/
-typedef struct { long long ll[ 6]; } XXH32_state_t;
-typedef struct { long long ll[11]; } XXH64_state_t;
+XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
/*
-These structures allow static allocation of XXH states.
-States must then be initialized using XXHnn_reset() before first use.
+These functions generate the xxHash of an input provided in multiple segments.
+Note that, for small input, they are slower than single-call functions, due to state management.
+For small input, prefer `XXH32()` and `XXH64()` .
-If you prefer dynamic allocation, please refer to functions below.
-*/
+XXH state must first be allocated, using XXH*_createState() .
-XXH32_state_t* XXH32_createState(void);
-XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
+Start a new hash by initializing state with a seed, using XXH*_reset().
-XXH64_state_t* XXH64_createState(void);
-XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
-
-/*
-These functions create and release memory for XXH state.
-States must then be initialized using XXHnn_reset() before first use.
-*/
+Then, feed the hash state by calling XXH*_update() as many times as necessary.
+Obviously, input must be allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+Finally, a hash value can be produced anytime, by using XXH*_digest().
+This function returns the nn-bits hash as an int or long long.
-XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed);
-XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-unsigned int XXH32_digest (const XXH32_state_t* statePtr);
+It's still possible to continue inserting input into the hash state after a digest,
+and generate some new hashes later on, by calling again XXH*_digest().
-XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
-XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
+When done, free XXH state space if it was allocated dynamically.
+*/
-/*
-These functions calculate the xxHash of an input provided in multiple smaller packets,
-as opposed to an input provided as a single block.
+/*====== Canonical representation ======*/
-XXH state space must first be allocated, using either static or dynamic method provided above.
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
-Start a new hash by initializing state with a seed, using XXHnn_reset().
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
-Then, feed the hash state by calling XXHnn_update() as many times as necessary.
-Obviously, input must be valid, meaning allocated and read accessible.
-The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
-Finally, you can produce a hash anytime, by using XXHnn_digest().
-This function returns the final nn-bits hash.
-You can nonetheless continue feeding the hash state with more input,
-and therefore get some new hashes, by calling again XXHnn_digest().
+#ifndef XXH_NO_LONG_LONG
+/*-**********************************************************************
+* 64-bits hash
+************************************************************************/
+typedef unsigned long long XXH64_hash_t;
-When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
+/*! XXH64() :
+ Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+ "seed" can be used to alter the result predictably.
+ This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
*/
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*====== Streaming ======*/
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
+XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
+
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
+
+/*====== Canonical representation ======*/
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+#endif /* XXH_NO_LONG_LONG */
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* ================================================================================================
+ This section contains definitions which are not guaranteed to remain stable.
+ They may change in future versions, becoming incompatible with a different version of the library.
+ They shall only be used with static linking.
+ Never use these definitions in association with dynamic linking !
+=================================================================================================== */
+
+/* These definitions are only meant to allow allocation of XXH state
+ statically, on stack, or in a struct for example.
+ Do not use members directly. */
+
+ struct XXH32_state_s {
+ unsigned total_len_32;
+ unsigned large_len;
+ unsigned v1;
+ unsigned v2;
+ unsigned v3;
+ unsigned v4;
+ unsigned mem32[4]; /* buffer defined as U32 for alignment */
+ unsigned memsize;
+ unsigned reserved; /* never read nor write, will be removed in a future version */
+ }; /* typedef'd to XXH32_state_t */
+
+#ifndef XXH_NO_LONG_LONG
+ struct XXH64_state_s {
+ unsigned long long total_len;
+ unsigned long long v1;
+ unsigned long long v2;
+ unsigned long long v3;
+ unsigned long long v4;
+ unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
+ unsigned memsize;
+ unsigned reserved[2]; /* never read nor write, will be removed in a future version */
+ }; /* typedef'd to XXH64_state_t */
+#endif
+
+# ifdef XXH_PRIVATE_API
+# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */
+# endif
+
+#endif /* XXH_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
+
+#endif /* XXHASH_H_5627135585666179 */
diff --git a/Externals/xxhash/xxhsum.c b/Externals/xxhash/xxhsum.c
index e090e51118..049bc7217b 100644
--- a/Externals/xxhash/xxhsum.c
+++ b/Externals/xxhash/xxhsum.c
@@ -1,71 +1,77 @@
/*
-bench.c - Demo program to benchmark open-source algorithm
-Copyright (C) Yann Collet 2012-2014
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along
-with this program; if not, write to the Free Software Foundation, Inc.,
-51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-You can contact the author at :
-- Blog homepage : http://fastcompression.blogspot.com/
-- Discussion group : https://groups.google.com/forum/?fromgroups#!forum/lz4c
+* xxhsum - Command line interface for xxhash algorithms
+* Copyright (C) Yann Collet 2012-2016
+*
+* GPL v2 License
+*
+* This program is free software; you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation; either version 2 of the License, or
+* (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License along
+* with this program; if not, write to the Free Software Foundation, Inc.,
+* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*
+* You can contact the author at :
+* - xxHash homepage : http://www.xxhash.com
+* - xxHash source repository : https://github.com/Cyan4973/xxHash
*/
-/**************************************
- * Compiler Options
- *************************************/
+/* xxhsum :
+ * Provides hash value of a file content, or a list of files, or stdin
+ * Display convention is Big Endian, for both 32 and 64 bits algorithms
+ */
+
+#ifndef XXHASH_C_2097394837
+#define XXHASH_C_2097394837
+
+/* ************************************
+* Compiler Options
+**************************************/
/* MS Visual */
#if defined(_MSC_VER) || defined(_WIN32)
# define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */
-# define BMK_LEGACY_TIMER 1 /* gettimeofday() not supported by MSVC */
#endif
/* Under Linux at least, pull in the *64 commands */
-#define _LARGEFILE64_SOURCE
+#ifndef _LARGEFILE64_SOURCE
+# define _LARGEFILE64_SOURCE
+#endif
-/**************************************
- * Includes
- *************************************/
-#include <stdlib.h> // malloc
-#include <stdio.h> // fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno
-#include <string.h> // strcmp
-#include <sys/types.h> // stat64
-#include <sys/stat.h> // stat64
+/* ************************************
+* Includes
+**************************************/
+#include <stdlib.h> /* malloc */
+#include <stdio.h> /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */
+#include <string.h> /* strcmp */
+#include <sys/types.h> /* stat64 */
+#include <sys/stat.h> /* stat64 */
+#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
+#define XXH_STATIC_LINKING_ONLY /* *_state_t */
#include "xxhash.h"
-/**************************************
- * OS-Specific Includes
- *************************************/
-// Use ftime() if gettimeofday() is not available on your target
-#if defined(BMK_LEGACY_TIMER)
-# include <sys/timeb.h> // timeb, ftime
-#else
-# include <sys/time.h> // gettimeofday
-#endif
-
+/*-************************************
+* OS-Specific Includes
+**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
-# include <fcntl.h> // _O_BINARY
-# include <io.h> // _setmode, _isatty
+# include <fcntl.h> /* _O_BINARY */
+# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
- int _fileno(FILE *stream); // MINGW somehow forgets to include this windows declaration into <stdio.h>
+ int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
-# include <unistd.h> // isatty, STDIN_FILENO
+# include <unistd.h> /* isatty, STDIN_FILENO */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(STDIN_FILENO)
#endif
@@ -75,128 +81,122 @@ You can contact the author at :
#endif
-/**************************************
- * Basic Types
- *************************************/
-#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99
-# include <stdint.h>
- typedef uint8_t BYTE;
- typedef uint16_t U16;
- typedef uint32_t U32;
- typedef int32_t S32;
- typedef uint64_t U64;
-#else
- typedef unsigned char BYTE;
- typedef unsigned short U16;
- typedef unsigned int U32;
- typedef signed int S32;
- typedef unsigned long long U64;
+/* ************************************
+* Basic Types
+**************************************/
+#ifndef MEM_MODULE
+# define MEM_MODULE
+# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+# else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+# endif
#endif
+static unsigned BMK_isLittleEndian(void)
+{
+ const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+}
-/**************************************
- * Constants
- *************************************/
-#define PROGRAM_NAME exename
-#define PROGRAM_VERSION ""
-#define COMPILED __DATE__
-#define AUTHOR "Yann Collet"
-#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s (%s) ***\n", PROGRAM_NAME, (int)(sizeof(void*)*8), PROGRAM_VERSION, AUTHOR, COMPILED
-
-#define NBLOOPS 3 // Default number of benchmark iterations
-#define TIMELOOP 2500 // Minimum timing per iteration
-#define PRIME 2654435761U
-#define KB *(1<<10)
-#define MB *(1<<20)
+/* *************************************
+* Constants
+***************************************/
+#define LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE
+#define QUOTE(str) #str
+#define EXPAND_AND_QUOTE(str) QUOTE(str)
+#define PROGRAM_VERSION EXPAND_AND_QUOTE(LIB_VERSION)
+static const int g_nbBits = (int)(sizeof(void*)*8);
+static const char g_lename[] = "little endian";
+static const char g_bename[] = "big endian";
+#define ENDIAN_NAME (BMK_isLittleEndian() ? g_lename : g_bename)
+static const char author[] = "Yann Collet";
+#define WELCOME_MESSAGE(exename) "%s %s (%i-bits %s), by %s \n", exename, PROGRAM_VERSION, g_nbBits, ENDIAN_NAME, author
+
+#define NBLOOPS 3 /* Default number of benchmark iterations */
+#define TIMELOOP_S 1
+#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* Minimum timing per iteration */
+#define XXHSUM32_DEFAULT_SEED 0 /* Default seed for algo_xxh32 */
+#define XXHSUM64_DEFAULT_SEED 0 /* Default seed for algo_xxh64 */
+
+#define KB *( 1<<10)
+#define MB *( 1<<20)
#define GB *(1U<<30)
#define MAX_MEM (2 GB - 64 MB)
static const char stdinName[] = "-";
+typedef enum { algo_xxh32, algo_xxh64 } algoType;
+static const algoType g_defaultAlgo = algo_xxh64; /* required within main() & usage() */
+
+/* <16 hex char> <SPC> <SPC> <filename> <'\0'>
+ * '4096' is typical Linux PATH_MAX configuration. */
+#define DEFAULT_LINE_LENGTH (sizeof(XXH64_hash_t) * 2 + 2 + 4096 + 1)
+
+/* Maximum acceptable line length. */
+#define MAX_LINE_LENGTH (32 KB)
-//**************************************
-// Display macros
-//**************************************
+/* ************************************
+* Display macros
+**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) DISPLAY(__VA_ARGS__);
-static unsigned g_displayLevel = 1;
-
-
-//**************************************
-// Unit variables
-//**************************************
-static int g_nbIterations = NBLOOPS;
-static int g_fn_selection = 1; // required within main() & usage()
-
-
-//*********************************************************
-// Benchmark Functions
-//*********************************************************
-
-#if defined(BMK_LEGACY_TIMER)
-
-static int BMK_GetMilliStart(void)
-{
- // Based on Legacy ftime()
- // Rolls over every ~ 12.1 days (0x100000/24/60/60)
- // Use GetMilliSpan to correct for rollover
- struct timeb tb;
- int nCount;
- ftime( &tb );
- nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
- return nCount;
-}
+static U32 g_displayLevel = 1;
-#else
-static int BMK_GetMilliStart(void)
-{
- // Based on newer gettimeofday()
- // Use GetMilliSpan to correct for rollover
- struct timeval tv;
- int nCount;
- gettimeofday(&tv, NULL);
- nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
- return nCount;
-}
+/* ************************************
+* Local variables
+**************************************/
+static size_t g_sampleSize = 100 KB;
+static U32 g_nbIterations = NBLOOPS;
-#endif
-static int BMK_GetMilliSpan( int nTimeStart )
+/* ************************************
+* Benchmark Functions
+**************************************/
+static clock_t BMK_clockSpan( clock_t start )
{
- int nSpan = BMK_GetMilliStart() - nTimeStart;
- if ( nSpan < 0 )
- nSpan += 0x100000 * 1000;
- return nSpan;
+ return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */
}
-static size_t BMK_findMaxMem(U64 requestedMem)
+static size_t BMK_findMaxMem(U64 requiredMem)
{
- size_t step = (64 MB);
- size_t allocatedMemory;
- BYTE* testmem=NULL;
+ size_t const step = 64 MB;
+ void* testmem = NULL;
- requestedMem += 3*step;
- requestedMem -= (size_t)requestedMem & (step-1);
- if (requestedMem > MAX_MEM) requestedMem = MAX_MEM;
- allocatedMemory = (size_t)requestedMem;
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
+ requiredMem += 2*step;
+ if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
- while (!testmem)
- {
- allocatedMemory -= step;
- testmem = (BYTE*) malloc((size_t)allocatedMemory);
+ while (!testmem) {
+ if (requiredMem > step) requiredMem -= step;
+ else requiredMem >>= 1;
+ testmem = malloc ((size_t)requiredMem);
}
free (testmem);
- return (size_t) (allocatedMemory - step);
+ /* keep some space available */
+ if (requiredMem > step) requiredMem -= step;
+ else requiredMem >>= 1;
+
+ return (size_t)requiredMem;
}
-static U64 BMK_GetFileSize(char* infilename)
+static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
@@ -206,194 +206,145 @@ static U64 BMK_GetFileSize(char* infilename)
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
- if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good...
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
+typedef U32 (*hashFunction)(const void* buffer, size_t bufferSize, U32 seed);
+
+static U32 localXXH32(const void* buffer, size_t bufferSize, U32 seed) { return XXH32(buffer, bufferSize, seed); }
+
+static U32 localXXH64(const void* buffer, size_t bufferSize, U32 seed) { return (U32)XXH64(buffer, bufferSize, seed); }
-static int BMK_benchFile(char** fileNamesTable, int nbFiles)
+static void BMK_benchHash(hashFunction h, const char* hName, const void* buffer, size_t bufferSize)
{
- int fileIdx=0;
- U32 hashResult=0;
+ static const U32 nbh_perloop = 100;
+ U32 iterationNb;
+ double fastestH = 100000000.;
+
+ DISPLAY("\r%79s\r", ""); /* Clean display line */
+ if (g_nbIterations<1) g_nbIterations=1;
+ for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) {
+ U32 nbHashes = 0, r=0;
+ clock_t cStart;
+
+ DISPLAY("%1i-%-17.17s : %10u ->\r", iterationNb, hName, (U32)bufferSize);
+ cStart = clock();
+ while (clock() == cStart); /* starts clock() at its exact beginning */
+ cStart = clock();
+
+ while (BMK_clockSpan(cStart) < TIMELOOP) {
+ U32 i;
+ for (i=0; i<nbh_perloop; i++)
+ r += h(buffer, bufferSize, i);
+ nbHashes += nbh_perloop;
+ }
+ if (r==0) DISPLAY(".\r"); /* need to do something with r to avoid compiler optimizing away the hash function */
+ { double const timeS = ((double)BMK_clockSpan(cStart) / CLOCKS_PER_SEC) / nbHashes;
+ if (timeS < fastestH) fastestH = timeS;
+ DISPLAY("%1i-%-17.17s : %10u -> %7.1f MB/s\r", iterationNb, hName, (U32)bufferSize, ((double)bufferSize / (1<<20)) / fastestH );
+ }
+ }
+ DISPLAY("%-19.19s : %10u -> %7.1f MB/s \n", hName, (U32)bufferSize, ((double)bufferSize / (1<<20)) / fastestH);
+}
- U64 totals = 0;
- double totalc = 0.;
+/* Note : buffer is supposed malloc'ed, hence aligned */
+static void BMK_benchMem(const void* buffer, size_t bufferSize)
+{
+ /* XXH32 bench */
+ BMK_benchHash(localXXH32, "XXH32", buffer, bufferSize);
- // Loop for each file
- while (fileIdx<nbFiles)
- {
- FILE* inFile;
- char* inFileName;
- U64 inFileSize;
- size_t benchedSize;
- size_t readSize;
- char* buffer;
- char* alignedBuffer;
-
- // Check file existence
- inFileName = fileNamesTable[fileIdx++];
- inFile = fopen( inFileName, "rb" );
- if (inFile==NULL)
- {
+ /* Bench XXH32 on Unaligned input */
+ if (bufferSize>1)
+ BMK_benchHash(localXXH32, "XXH32 unaligned", ((const char*)buffer)+1, bufferSize-1);
+
+ /* Bench XXH64 */
+ BMK_benchHash(localXXH64, "XXH64", buffer, bufferSize);
+
+ /* Bench XXH64 on Unaligned input */
+ if (bufferSize>1)
+ BMK_benchHash(localXXH64, "XXH64 unaligned", ((const char*)buffer)+1, bufferSize-1);
+}
+
+
+static size_t BMK_selectBenchedSize(const char* fileName)
+{ U64 const inFileSize = BMK_GetFileSize(fileName);
+ size_t benchedSize = (size_t) BMK_findMaxMem(inFileSize);
+ if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+ if (benchedSize < inFileSize) {
+ DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20));
+ }
+ return benchedSize;
+}
+
+
+static int BMK_benchFiles(const char** fileNamesTable, int nbFiles)
+{
+ int fileIdx;
+ for (fileIdx=0; fileIdx<nbFiles; fileIdx++) {
+ const char* const inFileName = fileNamesTable[fileIdx];
+ FILE* const inFile = fopen( inFileName, "rb" );
+ size_t const benchedSize = BMK_selectBenchedSize(inFileName);
+ char* const buffer = (char*)malloc(benchedSize+16);
+ void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes boundaries */
+
+ /* Checks */
+ if ((inFile==NULL) || (inFileName==NULL)) {
DISPLAY( "Pb opening %s\n", inFileName);
+ free(buffer);
return 11;
}
-
- // Memory allocation & restrictions
- inFileSize = BMK_GetFileSize(inFileName);
- benchedSize = (size_t) BMK_findMaxMem(inFileSize);
- if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
- if (benchedSize < inFileSize)
- {
- DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
- }
-
- buffer = (char*)malloc((size_t )benchedSize+16);
- if(!buffer)
- {
+ if(!buffer) {
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
}
- alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); // align on next 16 bytes boundaries
- // Fill input buffer
+ /* Fill input buffer */
DISPLAY("\rLoading %s... \n", inFileName);
- readSize = fread(alignedBuffer, 1, benchedSize, inFile);
- fclose(inFile);
-
- if(readSize != benchedSize)
- {
- DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
- free(buffer);
- return 13;
- }
-
-
- // Bench XXH32
- {
- int interationNb;
- double fastestC = 100000000.;
-
- DISPLAY("\r%79s\r", ""); // Clean display line
- for (interationNb = 1; interationNb <= g_nbIterations; interationNb++)
- {
- int nbHashes = 0;
- int milliTime;
-
- DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH32", (int)benchedSize);
-
- // Hash loop
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- int i;
- for (i=0; i<100; i++)
- {
- hashResult = XXH32(alignedBuffer, benchedSize, 0);
- nbHashes++;
- }
- }
- milliTime = BMK_GetMilliSpan(milliTime);
- if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes;
- DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000.);
- }
- DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X\n", "XXH32", (int)benchedSize, (double)benchedSize / fastestC / 1000., hashResult);
+ { size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile);
+ fclose(inFile);
+ if(readSize != benchedSize) {
+ DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
+ free(buffer);
+ return 13;
+ } }
- totals += benchedSize;
- totalc += fastestC;
- }
+ /* bench */
+ BMK_benchMem(alignedBuffer, benchedSize);
- // Bench Unaligned XXH32
- {
- int interationNb;
- double fastestC = 100000000.;
-
- DISPLAY("\r%79s\r", ""); // Clean display line
- for (interationNb = 1; (interationNb <= g_nbIterations) && ((benchedSize>1)); interationNb++)
- {
- int nbHashes = 0;
- int milliTime;
-
- DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "(unaligned)", (int)benchedSize);
- // Hash loop
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- int i;
- for (i=0; i<100; i++)
- {
- hashResult = XXH32(alignedBuffer+1, benchedSize-1, 0);
- nbHashes++;
- }
- }
- milliTime = BMK_GetMilliSpan(milliTime);
- if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes;
- DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH32 (unaligned)", (int)(benchedSize-1), (double)(benchedSize-1) / fastestC / 1000.);
- }
- DISPLAY("%-16.16s : %10i -> %7.1f MB/s \n", "XXH32 (unaligned)", (int)benchedSize-1, (double)(benchedSize-1) / fastestC / 1000.);
- }
+ free(buffer);
+ }
- // Bench XXH64
- {
- int interationNb;
- double fastestC = 100000000.;
- unsigned long long h64 = 0;
+ return 0;
+}
- DISPLAY("\r%79s\r", ""); // Clean display line
- for (interationNb = 1; interationNb <= g_nbIterations; interationNb++)
- {
- int nbHashes = 0;
- int milliTime;
-
- DISPLAY("%1i-%-14.14s : %10i ->\r", interationNb, "XXH64", (int)benchedSize);
-
- // Hash loop
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliStart() == milliTime);
- milliTime = BMK_GetMilliStart();
- while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
- {
- int i;
- for (i=0; i<100; i++)
- {
- h64 = XXH64(alignedBuffer, benchedSize, 0);
- nbHashes++;
- }
- }
- milliTime = BMK_GetMilliSpan(milliTime);
- if ((double)milliTime < fastestC*nbHashes) fastestC = (double)milliTime/nbHashes;
- DISPLAY("%1i-%-14.14s : %10i -> %7.1f MB/s\r", interationNb, "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000.);
- }
- DISPLAY("%-16.16s : %10i -> %7.1f MB/s 0x%08X%08X\n", "XXH64", (int)benchedSize, (double)benchedSize / fastestC / 1000., (U32)(h64>>32), (U32)(h64));
- totals += benchedSize;
- totalc += fastestC;
- }
- free(buffer);
+static int BMK_benchInternal(void)
+{
+ size_t const benchedSize = g_sampleSize;
+ void* const buffer = malloc(benchedSize);
+ if(!buffer) {
+ DISPLAY("\nError: not enough memory!\n");
+ return 12;
}
- if (nbFiles > 1)
- printf("%-16.16s :%11llu -> %7.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (double)totals/totalc/1000.);
+ /* bench */
+ DISPLAY("\rSample of %u KB... \n", (U32)(benchedSize >> 10));
+ BMK_benchMem(buffer, benchedSize);
+ free(buffer);
return 0;
}
-
static void BMK_checkResult(U32 r1, U32 r2)
{
static int nbTests = 1;
-
if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2);
- else
- {
+ else {
DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2);
exit(1);
}
@@ -404,22 +355,20 @@ static void BMK_checkResult(U32 r1, U32 r2)
static void BMK_checkResult64(U64 r1, U64 r2)
{
static int nbTests = 1;
-
- if (r1!=r2)
- {
+ if (r1!=r2) {
DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests);
- DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2<<32), (U32)r2);
+ DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2);
exit(1);
}
nbTests++;
}
-static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult)
+static void BMK_testSequence64(void* sentence, size_t len, U64 seed, U64 Nresult)
{
- U64 Dresult;
XXH64_state_t state;
- int index;
+ U64 Dresult;
+ size_t pos;
Dresult = XXH64(sentence, len, seed);
BMK_checkResult64(Dresult, Nresult);
@@ -430,28 +379,28 @@ static void BMK_testSequence64(void* sentence, int len, U64 seed, U64 Nresult)
BMK_checkResult64(Dresult, Nresult);
XXH64_reset(&state, seed);
- for (index=0; index<len; index++) XXH64_update(&state, ((char*)sentence)+index, 1);
+ for (pos=0; pos<len; pos++) XXH64_update(&state, ((char*)sentence)+pos, 1);
Dresult = XXH64_digest(&state);
BMK_checkResult64(Dresult, Nresult);
}
-static void BMK_testSequence(void* sentence, int len, U32 seed, U32 Nresult)
+static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult)
{
- U32 Dresult;
XXH32_state_t state;
- int index;
+ U32 Dresult;
+ size_t pos;
- Dresult = XXH32(sentence, len, seed);
+ Dresult = XXH32(sequence, len, seed);
BMK_checkResult(Dresult, Nresult);
XXH32_reset(&state, seed);
- XXH32_update(&state, sentence, len);
+ XXH32_update(&state, sequence, len);
Dresult = XXH32_digest(&state);
BMK_checkResult(Dresult, Nresult);
XXH32_reset(&state, seed);
- for (index=0; index<len; index++) XXH32_update(&state, ((char*)sentence)+index, 1);
+ for (pos=0; pos<len; pos++) XXH32_update(&state, ((const char*)sequence)+pos, 1);
Dresult = XXH32_digest(&state);
BMK_checkResult(Dresult, Nresult);
}
@@ -460,150 +409,712 @@ static void BMK_testSequence(void* sentence, int len, U32 seed, U32 Nresult)
#define SANITY_BUFFER_SIZE 101
static void BMK_sanityCheck(void)
{
+ static const U32 prime = 2654435761U;
BYTE sanityBuffer[SANITY_BUFFER_SIZE];
- int i;
- U32 prime = PRIME;
+ U32 byteGen = prime;
- for (i=0; i<SANITY_BUFFER_SIZE; i++)
- {
- sanityBuffer[i] = (BYTE)(prime>>24);
- prime *= prime;
+ int i;
+ for (i=0; i<SANITY_BUFFER_SIZE; i++) {
+ sanityBuffer[i] = (BYTE)(byteGen>>24);
+ byteGen *= byteGen;
}
BMK_testSequence(NULL, 0, 0, 0x02CC5D05);
- BMK_testSequence(NULL, 0, PRIME, 0x36B78AE7);
+ BMK_testSequence(NULL, 0, prime, 0x36B78AE7);
BMK_testSequence(sanityBuffer, 1, 0, 0xB85CBEE5);
- BMK_testSequence(sanityBuffer, 1, PRIME, 0xD5845D64);
+ BMK_testSequence(sanityBuffer, 1, prime, 0xD5845D64);
BMK_testSequence(sanityBuffer, 14, 0, 0xE5AA0AB4);
- BMK_testSequence(sanityBuffer, 14, PRIME, 0x4481951D);
+ BMK_testSequence(sanityBuffer, 14, prime, 0x4481951D);
BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x1F1AA412);
- BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0x498EC8E2);
+ BMK_testSequence(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0x498EC8E2);
BMK_testSequence64(NULL , 0, 0, 0xEF46DB3751D8E999ULL);
- BMK_testSequence64(NULL , 0, PRIME, 0xAC75FDA2929B17EFULL);
+ BMK_testSequence64(NULL , 0, prime, 0xAC75FDA2929B17EFULL);
BMK_testSequence64(sanityBuffer, 1, 0, 0x4FCE394CC88952D8ULL);
- BMK_testSequence64(sanityBuffer, 1, PRIME, 0x739840CB819FA723ULL);
+ BMK_testSequence64(sanityBuffer, 1, prime, 0x739840CB819FA723ULL);
BMK_testSequence64(sanityBuffer, 14, 0, 0xCFFA8DB881BC3A3DULL);
- BMK_testSequence64(sanityBuffer, 14, PRIME, 0x5B9611585EFCC9CBULL);
+ BMK_testSequence64(sanityBuffer, 14, prime, 0x5B9611585EFCC9CBULL);
BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL);
- BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, PRIME, 0xCAA65939306F1E21ULL);
+ BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL);
- DISPLAY("\r%79s\r", ""); // Clean display line
+ DISPLAY("\r%79s\r", ""); /* Clean display line */
DISPLAYLEVEL(2, "Sanity check -- all tests ok\n");
}
-static int BMK_hash(const char* fileName, U32 hashNb)
+/* ********************************************************
+* File Hashing
+**********************************************************/
+
+static void BMK_display_LittleEndian(const void* ptr, size_t length)
{
- FILE* inFile;
- size_t const blockSize = 64 KB;
+ const BYTE* p = (const BYTE*)ptr;
+ size_t idx;
+ for (idx=length-1; idx<length; idx--) /* intentional underflow to negative to detect end */
+ DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_display_BigEndian(const void* ptr, size_t length)
+{
+ const BYTE* p = (const BYTE*)ptr;
+ size_t idx;
+ for (idx=0; idx<length; idx++)
+ DISPLAYRESULT("%02x", p[idx]);
+}
+
+static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* inFile, void* buffer, size_t blockSize)
+{
+ XXH64_state_t state64;
+ XXH32_state_t state32;
size_t readSize;
- char* buffer;
- XXH64_state_t state;
- // Check file existence
- if (fileName == stdinName)
+ /* Init */
+ XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED);
+ XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED);
+
+ /* Load file & update hash */
+ readSize = 1;
+ while (readSize) {
+ readSize = fread(buffer, 1, blockSize, inFile);
+ switch(hashType)
+ {
+ case algo_xxh32:
+ XXH32_update(&state32, buffer, readSize);
+ break;
+ case algo_xxh64:
+ XXH64_update(&state64, buffer, readSize);
+ break;
+ default:
+ break;
+ }
+ }
+
+ switch(hashType)
{
+ case algo_xxh32:
+ { U32 const h32 = XXH32_digest(&state32);
+ memcpy(xxhHashValue, &h32, sizeof(h32));
+ break;
+ }
+ case algo_xxh64:
+ { U64 const h64 = XXH64_digest(&state64);
+ memcpy(xxhHashValue, &h64, sizeof(h64));
+ break;
+ }
+ default:
+ break;
+ }
+}
+
+
+typedef enum { big_endian, little_endian} endianess;
+
+static int BMK_hash(const char* fileName,
+ const algoType hashType,
+ const endianess displayEndianess)
+{
+ FILE* inFile;
+ size_t const blockSize = 64 KB;
+ void* buffer;
+ U32 h32 = 0;
+ U64 h64 = 0;
+
+ /* Check file existence */
+ if (fileName == stdinName) {
inFile = stdin;
SET_BINARY_MODE(stdin);
}
else
inFile = fopen( fileName, "rb" );
- if (inFile==NULL)
- {
+ if (inFile==NULL) {
DISPLAY( "Pb opening %s\n", fileName);
- return 11;
+ return 1;
}
- // Memory allocation & restrictions
- buffer = (char*)malloc(blockSize);
- if(!buffer)
- {
+ /* Memory allocation & restrictions */
+ buffer = malloc(blockSize);
+ if(!buffer) {
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
- return 12;
+ return 1;
}
- // Init
- switch(hashNb)
+ /* loading notification */
+ { const size_t fileNameSize = strlen(fileName);
+ const char* const fileNameEnd = fileName + fileNameSize;
+ const size_t maxInfoFilenameSize = fileNameSize > 30 ? 30 : fileNameSize;
+ size_t infoFilenameSize = 1;
+ while ( (infoFilenameSize < maxInfoFilenameSize)
+ &&(fileNameEnd[-1-infoFilenameSize] != '/')
+ &&(fileNameEnd[-1-infoFilenameSize] != '\\') )
+ infoFilenameSize++;
+ DISPLAY("\rLoading %s... \r", fileNameEnd - infoFilenameSize);
+ }
+
+ /* Load file & update hash */
+ switch(hashType)
{
- case 0:
- XXH32_reset((XXH32_state_t*)&state, 0);
+ case algo_xxh32:
+ BMK_hashStream(&h32, algo_xxh32, inFile, buffer, blockSize);
break;
- case 1:
- XXH64_reset(&state, 0);
+ case algo_xxh64:
+ BMK_hashStream(&h64, algo_xxh64, inFile, buffer, blockSize);
break;
default:
- DISPLAY("Error : bad hash algorithm ID\n");
- fclose(inFile);
- free(buffer);
- return -1;
+ break;
}
+ fclose(inFile);
+ free(buffer);
- // Load file & update hash
- DISPLAY("\rLoading %s... \r", fileName);
- readSize = 1;
- while (readSize)
+ /* display Hash */
+ switch(hashType)
{
- readSize = fread(buffer, 1, blockSize, inFile);
- switch(hashNb)
- {
- case 0:
- XXH32_update((XXH32_state_t*)&state, buffer, readSize);
+ case algo_xxh32:
+ { XXH32_canonical_t hcbe32;
+ XXH32_canonicalFromHash(&hcbe32, h32);
+ displayEndianess==big_endian ?
+ BMK_display_BigEndian(&hcbe32, sizeof(hcbe32)) : BMK_display_LittleEndian(&hcbe32, sizeof(hcbe32));
+ DISPLAYRESULT(" %s\n", fileName);
break;
- case 1:
- XXH64_update(&state, buffer, readSize);
+ }
+ case algo_xxh64:
+ { XXH64_canonical_t hcbe64;
+ XXH64_canonicalFromHash(&hcbe64, h64);
+ displayEndianess==big_endian ?
+ BMK_display_BigEndian(&hcbe64, sizeof(hcbe64)) : BMK_display_LittleEndian(&hcbe64, sizeof(hcbe64));
+ DISPLAYRESULT(" %s\n", fileName);
break;
- default:
+ }
+ default:
break;
+ }
+
+ return 0;
+}
+
+
+static int BMK_hashFiles(const char** fnList, int fnTotal,
+ algoType hashType, endianess displayEndianess)
+{
+ int fnNb;
+ int result = 0;
+
+ if (fnTotal==0)
+ return BMK_hash(stdinName, hashType, displayEndianess);
+
+ for (fnNb=0; fnNb<fnTotal; fnNb++)
+ result += BMK_hash(fnList[fnNb], hashType, displayEndianess);
+ DISPLAY("\r%70s\r", "");
+ return result;
+}
+
+
+typedef enum {
+ GetLine_ok,
+ GetLine_eof,
+ GetLine_exceedMaxLineLength,
+ GetLine_outOfMemory,
+} GetLineResult;
+
+typedef enum {
+ CanonicalFromString_ok,
+ CanonicalFromString_invalidFormat,
+} CanonicalFromStringResult;
+
+typedef enum {
+ ParseLine_ok,
+ ParseLine_invalidFormat,
+} ParseLineResult;
+
+typedef enum {
+ LineStatus_hashOk,
+ LineStatus_hashFailed,
+ LineStatus_failedToOpen,
+} LineStatus;
+
+typedef union {
+ XXH32_canonical_t xxh32;
+ XXH64_canonical_t xxh64;
+} Canonical;
+
+typedef struct {
+ Canonical canonical;
+ const char* filename;
+ int xxhBits; /* canonical type : 32:xxh32, 64:xxh64 */
+} ParsedLine;
+
+typedef struct {
+ unsigned long nProperlyFormattedLines;
+ unsigned long nImproperlyFormattedLines;
+ unsigned long nMismatchedChecksums;
+ unsigned long nOpenOrReadFailures;
+ unsigned long nMixedFormatLines;
+ int xxhBits;
+ int quit;
+} ParseFileReport;
+
+typedef struct {
+ const char* inFileName;
+ FILE* inFile;
+ int lineMax;
+ char* lineBuf;
+ size_t blockSize;
+ char* blockBuf;
+ int strictMode;
+ int statusOnly;
+ int warn;
+ int quiet;
+ ParseFileReport report;
+} ParseFileArg;
+
+
+/* Read line from stream.
+ Returns GetLine_ok, if it reads line successfully.
+ Returns GetLine_eof, if stream reaches EOF.
+ Returns GetLine_exceedMaxLineLength, if line length is longer than MAX_LINE_LENGTH.
+ Returns GetLine_outOfMemory, if line buffer memory allocation failed.
+ */
+static GetLineResult getLine(char** lineBuf, int* lineMax, FILE* inFile)
+{
+ GetLineResult result = GetLine_ok;
+ int len = 0;
+
+ if (*lineBuf == NULL || *lineMax < 1) {
+ *lineMax = DEFAULT_LINE_LENGTH;
+ *lineBuf = (char*) realloc(*lineBuf, *lineMax);
+ if(*lineBuf == NULL) return GetLine_outOfMemory;
+ }
+
+ for (;;) {
+ const int c = fgetc(inFile);
+ if (c == EOF) {
+ /* If we meet EOF before first character, returns GetLine_eof,
+ * otherwise GetLine_ok.
+ */
+ if (len == 0) result = GetLine_eof;
+ break;
+ }
+
+ /* Make enough space for len+1 (for final NUL) bytes. */
+ if (len+1 >= *lineMax) {
+ char* newLineBuf = NULL;
+ int newBufSize = *lineMax;
+
+ newBufSize += (newBufSize/2) + 1; /* x 1.5 */
+ if (newBufSize > MAX_LINE_LENGTH) newBufSize = MAX_LINE_LENGTH;
+ if (len+1 >= newBufSize) return GetLine_exceedMaxLineLength;
+
+ newLineBuf = (char*) realloc(*lineBuf, newBufSize);
+ if (newLineBuf == NULL) return GetLine_outOfMemory;
+
+ *lineBuf = newLineBuf;
+ *lineMax = newBufSize;
}
+
+ if (c == '\n') break;
+ (*lineBuf)[len++] = (char) c;
}
- fclose(inFile);
- free(buffer);
- // display Hash
- switch(hashNb)
+ (*lineBuf)[len] = '\0';
+ return result;
+}
+
+
+/* Converts one hexadecimal character to integer.
+ * Returns -1, if given character is not hexadecimal.
+ */
+static int charToHex(char c)
+{
+ int result = -1;
+ if (c >= '0' && c <= '9') {
+ result = (int) (c - '0');
+ } else if (c >= 'A' && c <= 'F') {
+ result = (int) (c - 'A') + 0x0a;
+ } else if (c >= 'a' && c <= 'f') {
+ result = (int) (c - 'a') + 0x0a;
+ }
+ return result;
+}
+
+
+/* Converts XXH32 canonical hexadecimal string hashStr to big endian unsigned char array dst.
+ * Returns CANONICAL_FROM_STRING_INVALID_FORMAT, if hashStr is not well formatted.
+ * Returns CANONICAL_FROM_STRING_OK, if hashStr is parsed successfully.
+ */
+static CanonicalFromStringResult canonicalFromString(unsigned char* dst,
+ size_t dstSize,
+ const char* hashStr)
+{
+ size_t i;
+ for (i = 0; i < dstSize; ++i) {
+ int h0, h1;
+
+ h0 = charToHex(hashStr[i*2 + 0]);
+ if (h0 < 0) return CanonicalFromString_invalidFormat;
+
+ h1 = charToHex(hashStr[i*2 + 1]);
+ if (h1 < 0) return CanonicalFromString_invalidFormat;
+
+ dst[i] = (unsigned char) ((h0 << 4) | h1);
+ }
+ return CanonicalFromString_ok;
+}
+
+
+/* Parse single line of xxHash checksum file.
+ * Returns PARSE_LINE_ERROR_INVALID_FORMAT, if line is not well formatted.
+ * Returns PARSE_LINE_OK if line is parsed successfully.
+ * And members of parseLine will be filled by parsed values.
+ *
+ * - line must be ended with '\0'.
+ * - Since parsedLine.filename will point within given argument `line`,
+ * users must keep `line`s content during they are using parsedLine.
+ *
+ * Given xxHash checksum line should have the following format:
+ *
+ * <8 or 16 hexadecimal char> <space> <space> <filename...> <'\0'>
+ */
+static ParseLineResult parseLine(ParsedLine* parsedLine, const char* line)
+{
+ const char* const firstSpace = strchr(line, ' ');
+ const char* const secondSpace = firstSpace + 1;
+
+ parsedLine->filename = NULL;
+ parsedLine->xxhBits = 0;
+
+ if (firstSpace == NULL || *secondSpace != ' ') return ParseLine_invalidFormat;
+
+ switch (firstSpace - line)
{
- case 0:
- {
- U32 h32 = XXH32_digest((XXH32_state_t*)&state);
- DISPLAYRESULT("%08x %s \n", h32, fileName);
+ case 8:
+ { XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32;
+ if (canonicalFromString(xxh32c->digest, sizeof(xxh32c->digest), line)
+ != CanonicalFromString_ok) {
+ return ParseLine_invalidFormat;
+ }
+ parsedLine->xxhBits = 32;
break;
}
- case 1:
- {
- U64 h64 = XXH64_digest(&state);
- DISPLAYRESULT("%08x%08x %s \n", (U32)(h64>>32), (U32)(h64), fileName);
+
+ case 16:
+ { XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64;
+ if (canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), line)
+ != CanonicalFromString_ok) {
+ return ParseLine_invalidFormat;
+ }
+ parsedLine->xxhBits = 64;
break;
}
+
default:
+ return ParseLine_invalidFormat;
break;
}
- return 0;
+ parsedLine->filename = secondSpace + 1;
+ return ParseLine_ok;
}
-//*********************************************************
-// Main
-//*********************************************************
+/*! Parse xxHash checksum file.
+ */
+static void parseFile1(ParseFileArg* parseFileArg)
+{
+ const char* const inFileName = parseFileArg->inFileName;
+ ParseFileReport* const report = &parseFileArg->report;
+
+ unsigned long lineNumber = 0;
+ memset(report, 0, sizeof(*report));
+
+ while (!report->quit) {
+ FILE* fp = NULL;
+ LineStatus lineStatus = LineStatus_hashFailed;
+ GetLineResult getLineResult;
+ ParsedLine parsedLine;
+ memset(&parsedLine, 0, sizeof(parsedLine));
+
+ lineNumber++;
+ if (lineNumber == 0) {
+ /* This is unlikely happen, but md5sum.c has this
+ * error check. */
+ DISPLAY("%s : too many checksum lines\n", inFileName);
+ report->quit = 1;
+ break;
+ }
+
+ getLineResult = getLine(&parseFileArg->lineBuf, &parseFileArg->lineMax,
+ parseFileArg->inFile);
+ if (getLineResult != GetLine_ok) {
+ if (getLineResult == GetLine_eof) break;
+
+ switch (getLineResult)
+ {
+ case GetLine_ok:
+ case GetLine_eof:
+ /* These cases never happen. See above getLineResult related "if"s.
+ They exist just for make gcc's -Wswitch-enum happy. */
+ break;
+
+ default:
+ DISPLAY("%s : %lu: unknown error\n", inFileName, lineNumber);
+ break;
+
+ case GetLine_exceedMaxLineLength:
+ DISPLAY("%s : %lu: too long line\n", inFileName, lineNumber);
+ break;
+
+ case GetLine_outOfMemory:
+ DISPLAY("%s : %lu: out of memory\n", inFileName, lineNumber);
+ break;
+ }
+ report->quit = 1;
+ break;
+ }
+
+ if (parseLine(&parsedLine, parseFileArg->lineBuf) != ParseLine_ok) {
+ report->nImproperlyFormattedLines++;
+ if (parseFileArg->warn) {
+ DISPLAY("%s : %lu: improperly formatted XXHASH checksum line\n"
+ , inFileName, lineNumber);
+ }
+ continue;
+ }
+
+ if (report->xxhBits != 0 && report->xxhBits != parsedLine.xxhBits) {
+ /* Don't accept xxh32/xxh64 mixed file */
+ report->nImproperlyFormattedLines++;
+ report->nMixedFormatLines++;
+ if (parseFileArg->warn) {
+ DISPLAY("%s : %lu: improperly formatted XXHASH checksum line (XXH32/64)\n"
+ , inFileName, lineNumber);
+ }
+ continue;
+ }
+
+ report->nProperlyFormattedLines++;
+ if (report->xxhBits == 0) {
+ report->xxhBits = parsedLine.xxhBits;
+ }
+
+ fp = fopen(parsedLine.filename, "rb");
+ if (fp == NULL) {
+ lineStatus = LineStatus_failedToOpen;
+ } else {
+ lineStatus = LineStatus_hashFailed;
+ switch (parsedLine.xxhBits)
+ {
+ case 32:
+ { XXH32_hash_t xxh;
+ BMK_hashStream(&xxh, algo_xxh32, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+ if (xxh == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) {
+ lineStatus = LineStatus_hashOk;
+ } }
+ break;
+
+ case 64:
+ { XXH64_hash_t xxh;
+ BMK_hashStream(&xxh, algo_xxh64, fp, parseFileArg->blockBuf, parseFileArg->blockSize);
+ if (xxh == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) {
+ lineStatus = LineStatus_hashOk;
+ } }
+ break;
+
+ default:
+ break;
+ }
+ fclose(fp);
+ }
+
+ switch (lineStatus)
+ {
+ default:
+ DISPLAY("%s : unknown error\n", inFileName);
+ report->quit = 1;
+ break;
+
+ case LineStatus_failedToOpen:
+ report->nOpenOrReadFailures++;
+ if (!parseFileArg->statusOnly) {
+ DISPLAYRESULT("%s : %lu: FAILED open or read %s\n"
+ , inFileName, lineNumber, parsedLine.filename);
+ }
+ break;
+
+ case LineStatus_hashOk:
+ case LineStatus_hashFailed:
+ { int b = 1;
+ if (lineStatus == LineStatus_hashOk) {
+ /* If --quiet is specified, don't display "OK" */
+ if (parseFileArg->quiet) b = 0;
+ } else {
+ report->nMismatchedChecksums++;
+ }
+
+ if (b && !parseFileArg->statusOnly) {
+ DISPLAYRESULT("%s: %s\n", parsedLine.filename
+ , lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
+ } }
+ break;
+ }
+ } /* while (!report->quit) */
+}
+
+
+/* Parse xxHash checksum file.
+ * Returns 1, if all procedures were succeeded.
+ * Returns 0, if any procedures was failed.
+ *
+ * If strictMode != 0, return error code if any line is invalid.
+ * If statusOnly != 0, don't generate any output.
+ * If warn != 0, print a warning message to stderr.
+ * If quiet != 0, suppress "OK" line.
+ *
+ * "All procedures are succeeded" means:
+ * - Checksum file contains at least one line and less than SIZE_T_MAX lines.
+ * - All files are properly opened and read.
+ * - All hash values match with its content.
+ * - (strict mode) All lines in checksum file are consistent and well formatted.
+ *
+ */
+static int checkFile(const char* inFileName,
+ const endianess displayEndianess,
+ U32 strictMode,
+ U32 statusOnly,
+ U32 warn,
+ U32 quiet)
+{
+ int result = 0;
+ FILE* inFile = NULL;
+ ParseFileArg parseFileArgBody;
+ ParseFileArg* const parseFileArg = &parseFileArgBody;
+ ParseFileReport* const report = &parseFileArg->report;
+
+ if (displayEndianess != big_endian) {
+ /* Don't accept little endian */
+ DISPLAY( "Check file mode doesn't support little endian\n" );
+ return 0;
+ }
+
+ /* note : stdinName is special constant pointer. It is not a string. */
+ if (inFileName == stdinName) {
+ /* note : Since we expect text input for xxhash -c mode,
+ * Don't set binary mode for stdin */
+ inFile = stdin;
+ } else {
+ inFile = fopen( inFileName, "rt" );
+ }
+
+ if (inFile == NULL) {
+ DISPLAY( "Pb opening %s\n", inFileName);
+ return 0;
+ }
+
+ parseFileArg->inFileName = inFileName;
+ parseFileArg->inFile = inFile;
+ parseFileArg->lineMax = DEFAULT_LINE_LENGTH;
+ parseFileArg->lineBuf = (char*) malloc((size_t) parseFileArg->lineMax);
+ parseFileArg->blockSize = 64 * 1024;
+ parseFileArg->blockBuf = (char*) malloc(parseFileArg->blockSize);
+ parseFileArg->strictMode = strictMode;
+ parseFileArg->statusOnly = statusOnly;
+ parseFileArg->warn = warn;
+ parseFileArg->quiet = quiet;
+
+ parseFile1(parseFileArg);
+
+ free(parseFileArg->blockBuf);
+ free(parseFileArg->lineBuf);
+
+ if (inFile != stdin) fclose(inFile);
+
+ /* Show error/warning messages. All messages are copied from md5sum.c
+ */
+ if (report->nProperlyFormattedLines == 0) {
+ DISPLAY("%s: no properly formatted XXHASH checksum lines found\n", inFileName);
+ } else if (!statusOnly) {
+ if (report->nImproperlyFormattedLines) {
+ DISPLAYRESULT("%lu lines are improperly formatted\n"
+ , report->nImproperlyFormattedLines);
+ }
+ if (report->nOpenOrReadFailures) {
+ DISPLAYRESULT("%lu listed files could not be read\n"
+ , report->nOpenOrReadFailures);
+ }
+ if (report->nMismatchedChecksums) {
+ DISPLAYRESULT("%lu computed checksums did NOT match\n"
+ , report->nMismatchedChecksums);
+ } }
+
+ /* Result (exit) code logic is copied from
+ * gnu coreutils/src/md5sum.c digest_check() */
+ result = report->nProperlyFormattedLines != 0
+ && report->nMismatchedChecksums == 0
+ && report->nOpenOrReadFailures == 0
+ && (!strictMode || report->nImproperlyFormattedLines == 0)
+ && report->quit == 0;
+ return result;
+}
+
+
+static int checkFiles(const char** fnList, int fnTotal,
+ const endianess displayEndianess,
+ U32 strictMode,
+ U32 statusOnly,
+ U32 warn,
+ U32 quiet)
+{
+ int ok = 1;
+
+ /* Special case for stdinName "-",
+ * note: stdinName is not a string. It's special pointer. */
+ if (fnTotal==0) {
+ ok &= checkFile(stdinName, displayEndianess, strictMode, statusOnly, warn, quiet);
+ } else {
+ int fnNb;
+ for (fnNb=0; fnNb<fnTotal; fnNb++)
+ ok &= checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, warn, quiet);
+ }
+ return ok ? 0 : 1;
+}
+
+
+/* ********************************************************
+* Main
+**********************************************************/
static int usage(const char* exename)
{
- DISPLAY( WELCOME_MESSAGE );
+ DISPLAY( WELCOME_MESSAGE(exename) );
DISPLAY( "Usage :\n");
- DISPLAY( " %s [arg] [filename]\n", exename);
+ DISPLAY( " %s [arg] [filenames]\n", exename);
DISPLAY( "When no filename provided, or - provided : use stdin as input\n");
DISPLAY( "Arguments :\n");
- DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default %i)\n", g_fn_selection);
+ DISPLAY( " -H# : hash selection : 0=32bits, 1=64bits (default: %i)\n", (int)g_defaultAlgo);
+ DISPLAY( " -c : read xxHash sums from the [filenames] and check them\n");
+ DISPLAY( " -h : help \n");
+ return 0;
+}
+
+
+static int usage_advanced(const char* exename)
+{
+ usage(exename);
+ DISPLAY( "Advanced :\n");
+ DISPLAY( " --little-endian : hash printed using little endian convention (default: big endian)\n");
+ DISPLAY( " -V, --version : display version\n");
+ DISPLAY( " -h, --help : display long help and exit\n");
DISPLAY( " -b : benchmark mode \n");
DISPLAY( " -i# : number of iterations (benchmark mode; default %i)\n", g_nbIterations);
- DISPLAY( " -h : help (this text)\n");
+ DISPLAY( "\n");
+ DISPLAY( "The following four options are useful only when verifying checksums (-c):\n");
+ DISPLAY( "--strict : don't print OK for each successfully verified file\n");
+ DISPLAY( "--status : don't output anything, status code shows success\n");
+ DISPLAY( "--quiet : exit non-zero for improperly formatted checksum lines\n");
+ DISPLAY( "--warn : warn about improperly formatted checksum lines\n");
return 0;
}
-
static int badusage(const char* exename)
{
DISPLAY("Wrong parameters\n");
@@ -612,78 +1123,116 @@ static int badusage(const char* exename)
}
-int main(int argc, char** argv)
+int main(int argc, const char** argv)
{
int i, filenamesStart=0;
- const char* input_filename = (char*)stdinName;
- const char* exename = argv[0];
+ const char* const exename = argv[0];
U32 benchmarkMode = 0;
-
- // xxh32sum default to 32 bits checksum
- if (strstr(exename, "xxh32sum")!=NULL) g_fn_selection=0;
-
- for(i=1; i<argc; i++)
- {
- char* argument = argv[i];
-
- if(!argument) continue; // Protection if argument empty
-
- if (*argument!='-')
- {
- input_filename=argument;
- if (filenamesStart==0) filenamesStart=i;
+ U32 fileCheckMode = 0;
+ U32 strictMode = 0;
+ U32 statusOnly = 0;
+ U32 warn = 0;
+ U32 quiet = 0;
+ algoType algo = g_defaultAlgo;
+ endianess displayEndianess = big_endian;
+
+ /* special case : xxh32sum default to 32 bits checksum */
+ if (strstr(exename, "xxh32sum") != NULL) algo = algo_xxh32;
+
+ for(i=1; i<argc; i++) {
+ const char* argument = argv[i];
+
+ if(!argument) continue; /* Protection, if argument empty */
+
+ if (!strcmp(argument, "--little-endian")) { displayEndianess = little_endian; continue; }
+ if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; }
+ if (!strcmp(argument, "--strict")) { strictMode = 1; continue; }
+ if (!strcmp(argument, "--status")) { statusOnly = 1; continue; }
+ if (!strcmp(argument, "--quiet")) { quiet = 1; continue; }
+ if (!strcmp(argument, "--warn")) { warn = 1; continue; }
+ if (!strcmp(argument, "--help")) { return usage_advanced(exename); }
+ if (!strcmp(argument, "--version")) { DISPLAY(WELCOME_MESSAGE(exename)); return 0; }
+
+ if (*argument!='-') {
+ if (filenamesStart==0) filenamesStart=i; /* only supports a continuous list of filenames */
continue;
}
- // Select command
- // note : *argument=='-'
- argument++;
+ /* command selection */
+ argument++; /* note : *argument=='-' */
- while (*argument!=0)
- {
+ while (*argument!=0) {
switch(*argument)
{
- // Display help on usage
+ /* Display version */
+ case 'V':
+ DISPLAY(WELCOME_MESSAGE(exename)); return 0;
+
+ /* Display help on usage */
case 'h':
- return usage(exename);
+ return usage_advanced(exename);
- // select hash algorithm
+ /* select hash algorithm */
case 'H':
- g_fn_selection = argument[1] - '0';
+ algo = (algoType)(argument[1] - '0');
argument+=2;
break;
- // Trigger benchmark mode
+ /* File check mode */
+ case 'c':
+ fileCheckMode=1;
+ argument++;
+ break;
+
+ /* Warning mode (file check mode only, alias of "--warning") */
+ case 'w':
+ warn=1;
+ argument++;
+ break;
+
+ /* Trigger benchmark mode */
case 'b':
argument++;
benchmarkMode=1;
break;
- // Modify Nb Iterations (benchmark only)
+ /* Modify Nb Iterations (benchmark only) */
case 'i':
g_nbIterations = argument[1] - '0';
argument+=2;
break;
+ /* Modify Block size (benchmark only) */
+ case 'B':
+ argument++;
+ g_sampleSize = 0;
+ while (argument[0]>='0' && argument[0]<='9')
+ g_sampleSize *= 10, g_sampleSize += argument[0]-'0', argument++;
+ break;
+
default:
return badusage(exename);
}
}
- }
+ } /* for(i=1; i<argc; i++) */
- // Check if input is defined as console; trigger an error in this case
- if ((input_filename == stdinName) && IS_CONSOLE(stdin) ) return badusage(exename);
-
- // Check results are good
- if (benchmarkMode)
- {
- if (filenamesStart==0) return badusage(exename);
- DISPLAY( WELCOME_MESSAGE );
+ /* Check benchmark mode */
+ if (benchmarkMode) {
+ DISPLAY( WELCOME_MESSAGE(exename) );
BMK_sanityCheck();
- return BMK_benchFile(argv+filenamesStart, argc-filenamesStart);
+ if (filenamesStart==0) return BMK_benchInternal();
+ return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart);
}
- if(g_fn_selection < 0 || g_fn_selection > 1) return badusage(exename);
+ /* Check if input is defined as console; trigger an error in this case */
+ if ( (filenamesStart==0) && IS_CONSOLE(stdin) ) return badusage(exename);
- return BMK_hash(input_filename, g_fn_selection);
+ if (filenamesStart==0) filenamesStart = argc;
+ if (fileCheckMode) {
+ return checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianess, strictMode, statusOnly, warn, quiet);
+ } else {
+ return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess);
+ }
}
+
+#endif /* XXHASH_C_2097394837 */
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-07 17:10:56 +0000