1afc2ba1dSToomas Soome /* 2afc2ba1dSToomas Soome * LZ4 - Fast LZ compression algorithm 3afc2ba1dSToomas Soome * Header File 4afc2ba1dSToomas Soome * Copyright (C) 2011-2013, Yann Collet. 5afc2ba1dSToomas Soome * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6afc2ba1dSToomas Soome * 7afc2ba1dSToomas Soome * Redistribution and use in source and binary forms, with or without 8afc2ba1dSToomas Soome * modification, are permitted provided that the following conditions are 9afc2ba1dSToomas Soome * met: 10afc2ba1dSToomas Soome * 11afc2ba1dSToomas Soome * * Redistributions of source code must retain the above copyright 12afc2ba1dSToomas Soome * notice, this list of conditions and the following disclaimer. 13afc2ba1dSToomas Soome * * Redistributions in binary form must reproduce the above 14afc2ba1dSToomas Soome * copyright notice, this list of conditions and the following disclaimer 15afc2ba1dSToomas Soome * in the documentation and/or other materials provided with the 16afc2ba1dSToomas Soome * distribution. 17afc2ba1dSToomas Soome * 18afc2ba1dSToomas Soome * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19afc2ba1dSToomas Soome * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20afc2ba1dSToomas Soome * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21afc2ba1dSToomas Soome * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22afc2ba1dSToomas Soome * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23afc2ba1dSToomas Soome * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24afc2ba1dSToomas Soome * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25afc2ba1dSToomas Soome * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26afc2ba1dSToomas Soome * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27afc2ba1dSToomas Soome * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28afc2ba1dSToomas Soome * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29afc2ba1dSToomas Soome * 30afc2ba1dSToomas Soome * You can contact the author at : 31afc2ba1dSToomas Soome * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html 32afc2ba1dSToomas Soome * - LZ4 source repository : http://code.google.com/p/lz4/ 33afc2ba1dSToomas Soome */ 34*10ae99eeSToomas Soome /* 35*10ae99eeSToomas Soome * Copyright (c) 2016 by Delphix. All rights reserved. 36*10ae99eeSToomas Soome */ 37afc2ba1dSToomas Soome 38*10ae99eeSToomas Soome #if defined(_KERNEL) || defined(_FAKE_KERNEL) 39*10ae99eeSToomas Soome #include <sys/zfs_context.h> 40*10ae99eeSToomas Soome #elif defined(_STANDALONE) 41*10ae99eeSToomas Soome #include <sys/cdefs.h> 42*10ae99eeSToomas Soome #include <stand.h> 43afc2ba1dSToomas Soome #include <sys/types.h> 44*10ae99eeSToomas Soome #include <sys/endian.h> 45afc2ba1dSToomas Soome #include <assert.h> 46*10ae99eeSToomas Soome 47*10ae99eeSToomas Soome #define ASSERT assert 48*10ae99eeSToomas Soome #else 49afc2ba1dSToomas Soome #include <string.h> 50*10ae99eeSToomas Soome #include <stdlib.h> 51*10ae99eeSToomas Soome #include <sys/types.h> 52*10ae99eeSToomas Soome #include <sys/sysmacros.h> 53*10ae99eeSToomas Soome #include <netinet/in.h> 54*10ae99eeSToomas Soome #include <assert.h> 55*10ae99eeSToomas Soome 56*10ae99eeSToomas Soome #define ASSERT assert 57*10ae99eeSToomas Soome #endif 58*10ae99eeSToomas Soome #include <lz4.h> 59afc2ba1dSToomas Soome 60afc2ba1dSToomas Soome static int real_LZ4_compress(const char *source, char *dest, int isize, 61afc2ba1dSToomas Soome int osize); 62afc2ba1dSToomas Soome static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest, 63afc2ba1dSToomas Soome int isize, int maxOutputSize); 64afc2ba1dSToomas Soome static int LZ4_compressCtx(void *ctx, const char *source, char *dest, 65afc2ba1dSToomas Soome int isize, int osize); 66afc2ba1dSToomas Soome static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest, 67afc2ba1dSToomas Soome int isize, int osize); 68afc2ba1dSToomas Soome 69afc2ba1dSToomas Soome size_t 70*10ae99eeSToomas Soome lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, 71*10ae99eeSToomas Soome int n __unused) 72afc2ba1dSToomas Soome { 73afc2ba1dSToomas Soome uint32_t bufsiz; 74afc2ba1dSToomas Soome char *dest = d_start; 75afc2ba1dSToomas Soome 76*10ae99eeSToomas Soome ASSERT(d_len >= sizeof (bufsiz)); 77afc2ba1dSToomas Soome 78afc2ba1dSToomas Soome bufsiz = real_LZ4_compress(s_start, &dest[sizeof (bufsiz)], s_len, 79afc2ba1dSToomas Soome d_len - sizeof (bufsiz)); 80afc2ba1dSToomas Soome 81afc2ba1dSToomas Soome /* Signal an error if the compression routine returned zero. */ 82afc2ba1dSToomas Soome if (bufsiz == 0) 83afc2ba1dSToomas Soome return (s_len); 84afc2ba1dSToomas Soome 85afc2ba1dSToomas Soome /* 86afc2ba1dSToomas Soome * Encode the compresed buffer size at the start. We'll need this in 87afc2ba1dSToomas Soome * decompression to counter the effects of padding which might be 88afc2ba1dSToomas Soome * added to the compressed buffer and which, if unhandled, would 89afc2ba1dSToomas Soome * confuse the hell out of our decompression function. 90afc2ba1dSToomas Soome */ 91*10ae99eeSToomas Soome #if defined(_KERNEL) || defined(_FAKE_KERNEL) 92*10ae99eeSToomas Soome *(uint32_t *)(void *)dest = BE_32(bufsiz); 93*10ae99eeSToomas Soome #else 94*10ae99eeSToomas Soome *(uint32_t *)(void *)dest = htonl(bufsiz); 95*10ae99eeSToomas Soome #endif 96afc2ba1dSToomas Soome 97afc2ba1dSToomas Soome return (bufsiz + sizeof (bufsiz)); 98afc2ba1dSToomas Soome } 99afc2ba1dSToomas Soome 100afc2ba1dSToomas Soome int 101*10ae99eeSToomas Soome lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, 102*10ae99eeSToomas Soome int n __unused) 103afc2ba1dSToomas Soome { 104afc2ba1dSToomas Soome const char *src = s_start; 105*10ae99eeSToomas Soome #if defined(_KERNEL) || defined(_FAKE_KERNEL) 106*10ae99eeSToomas Soome uint32_t bufsiz = BE_IN32(s_start); 107*10ae99eeSToomas Soome #else 108*10ae99eeSToomas Soome uint32_t bufsiz = htonl(*(uint32_t *)s_start); 109*10ae99eeSToomas Soome #endif 110afc2ba1dSToomas Soome 111afc2ba1dSToomas Soome /* invalid compressed buffer size encoded at start */ 112afc2ba1dSToomas Soome if (bufsiz + sizeof (bufsiz) > s_len) 113afc2ba1dSToomas Soome return (1); 114afc2ba1dSToomas Soome 115afc2ba1dSToomas Soome /* 116afc2ba1dSToomas Soome * Returns 0 on success (decompression function returned non-negative) 117*10ae99eeSToomas Soome * and non-zero on failure (decompression function returned negative). 118afc2ba1dSToomas Soome */ 119afc2ba1dSToomas Soome return (LZ4_uncompress_unknownOutputSize(&src[sizeof (bufsiz)], 120afc2ba1dSToomas Soome d_start, bufsiz, d_len) < 0); 121afc2ba1dSToomas Soome } 122afc2ba1dSToomas Soome 123afc2ba1dSToomas Soome /* 124afc2ba1dSToomas Soome * LZ4 API Description: 125afc2ba1dSToomas Soome * 126afc2ba1dSToomas Soome * Simple Functions: 127afc2ba1dSToomas Soome * real_LZ4_compress() : 128afc2ba1dSToomas Soome * isize : is the input size. Max supported value is ~1.9GB 129afc2ba1dSToomas Soome * return : the number of bytes written in buffer dest 130*10ae99eeSToomas Soome * or 0 if the compression fails (if LZ4_COMPRESSMIN is set). 131afc2ba1dSToomas Soome * note : destination buffer must be already allocated. 132afc2ba1dSToomas Soome * destination buffer must be sized to handle worst cases 133*10ae99eeSToomas Soome * situations (input data not compressible). 134afc2ba1dSToomas Soome * 135afc2ba1dSToomas Soome * Advanced Functions 136afc2ba1dSToomas Soome * 137*10ae99eeSToomas Soome * LZ4_compressBound() : 138*10ae99eeSToomas Soome * Provides the maximum size that LZ4 may output in a "worst case" 139*10ae99eeSToomas Soome * scenario (input data not compressible) primarily useful for memory 140*10ae99eeSToomas Soome * allocation of output buffer. 141*10ae99eeSToomas Soome * 142*10ae99eeSToomas Soome * isize : is the input size. Max supported value is ~1.9GB 143*10ae99eeSToomas Soome * return : maximum output size in a "worst case" scenario 144*10ae99eeSToomas Soome * note : this function is limited by "int" range (2^31-1) 145*10ae99eeSToomas Soome * 146afc2ba1dSToomas Soome * LZ4_uncompress_unknownOutputSize() : 147afc2ba1dSToomas Soome * isize : is the input size, therefore the compressed size 148afc2ba1dSToomas Soome * maxOutputSize : is the size of the destination buffer (which must be 149afc2ba1dSToomas Soome * already allocated) 150afc2ba1dSToomas Soome * return : the number of bytes decoded in the destination buffer 151afc2ba1dSToomas Soome * (necessarily <= maxOutputSize). If the source stream is 152afc2ba1dSToomas Soome * malformed, the function will stop decoding and return a 153afc2ba1dSToomas Soome * negative result, indicating the byte position of the faulty 154afc2ba1dSToomas Soome * instruction. This function never writes beyond dest + 155afc2ba1dSToomas Soome * maxOutputSize, and is therefore protected against malicious 156afc2ba1dSToomas Soome * data packets. 157afc2ba1dSToomas Soome * note : Destination buffer must be already allocated. 158afc2ba1dSToomas Soome * 159afc2ba1dSToomas Soome * LZ4_compressCtx() : 160afc2ba1dSToomas Soome * This function explicitly handles the CTX memory structure. 161afc2ba1dSToomas Soome * 162afc2ba1dSToomas Soome * ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated 163afc2ba1dSToomas Soome * by the caller (either on the stack or using kmem_zalloc). Passing NULL 164afc2ba1dSToomas Soome * isn't valid. 165afc2ba1dSToomas Soome * 166afc2ba1dSToomas Soome * LZ4_compress64kCtx() : 167afc2ba1dSToomas Soome * Same as LZ4_compressCtx(), but specific to small inputs (<64KB). 168afc2ba1dSToomas Soome * isize *Must* be <64KB, otherwise the output will be corrupted. 169afc2ba1dSToomas Soome * 170afc2ba1dSToomas Soome * ILLUMOS CHANGES: the CTX memory structure must be explicitly allocated 171afc2ba1dSToomas Soome * by the caller (either on the stack or using kmem_zalloc). Passing NULL 172afc2ba1dSToomas Soome * isn't valid. 173afc2ba1dSToomas Soome */ 174afc2ba1dSToomas Soome 175afc2ba1dSToomas Soome /* 176afc2ba1dSToomas Soome * Tuning parameters 177afc2ba1dSToomas Soome */ 178afc2ba1dSToomas Soome 179afc2ba1dSToomas Soome /* 180afc2ba1dSToomas Soome * COMPRESSIONLEVEL: Increasing this value improves compression ratio 181afc2ba1dSToomas Soome * Lowering this value reduces memory usage. Reduced memory usage 182afc2ba1dSToomas Soome * typically improves speed, due to cache effect (ex: L1 32KB for Intel, 183afc2ba1dSToomas Soome * L1 64KB for AMD). Memory usage formula : N->2^(N+2) Bytes 184afc2ba1dSToomas Soome * (examples : 12 -> 16KB ; 17 -> 512KB) 185afc2ba1dSToomas Soome */ 186afc2ba1dSToomas Soome #define COMPRESSIONLEVEL 12 187afc2ba1dSToomas Soome 188afc2ba1dSToomas Soome /* 189afc2ba1dSToomas Soome * NOTCOMPRESSIBLE_CONFIRMATION: Decreasing this value will make the 190afc2ba1dSToomas Soome * algorithm skip faster data segments considered "incompressible". 191afc2ba1dSToomas Soome * This may decrease compression ratio dramatically, but will be 192afc2ba1dSToomas Soome * faster on incompressible data. Increasing this value will make 193afc2ba1dSToomas Soome * the algorithm search more before declaring a segment "incompressible". 194afc2ba1dSToomas Soome * This could improve compression a bit, but will be slower on 195afc2ba1dSToomas Soome * incompressible data. The default value (6) is recommended. 196afc2ba1dSToomas Soome */ 197afc2ba1dSToomas Soome #define NOTCOMPRESSIBLE_CONFIRMATION 6 198afc2ba1dSToomas Soome 199afc2ba1dSToomas Soome /* 200afc2ba1dSToomas Soome * BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE: This will provide a boost to 201afc2ba1dSToomas Soome * performance for big endian cpu, but the resulting compressed stream 202afc2ba1dSToomas Soome * will be incompatible with little-endian CPU. You can set this option 203afc2ba1dSToomas Soome * to 1 in situations where data will stay within closed environment. 204afc2ba1dSToomas Soome * This option is useless on Little_Endian CPU (such as x86). 205afc2ba1dSToomas Soome */ 206afc2ba1dSToomas Soome /* #define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 */ 207afc2ba1dSToomas Soome 208afc2ba1dSToomas Soome /* 209afc2ba1dSToomas Soome * CPU Feature Detection 210afc2ba1dSToomas Soome */ 211afc2ba1dSToomas Soome 212afc2ba1dSToomas Soome /* 32 or 64 bits ? */ 213afc2ba1dSToomas Soome #if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || \ 214afc2ba1dSToomas Soome defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || \ 215afc2ba1dSToomas Soome defined(__LP64__) || defined(_LP64)) 216afc2ba1dSToomas Soome #define LZ4_ARCH64 1 217afc2ba1dSToomas Soome #else 218afc2ba1dSToomas Soome #define LZ4_ARCH64 0 219afc2ba1dSToomas Soome #endif 220afc2ba1dSToomas Soome 221afc2ba1dSToomas Soome /* 222afc2ba1dSToomas Soome * Limits the amount of stack space that the algorithm may consume to hold 223afc2ba1dSToomas Soome * the compression lookup table. The value `9' here means we'll never use 224afc2ba1dSToomas Soome * more than 2k of stack (see above for a description of COMPRESSIONLEVEL). 225afc2ba1dSToomas Soome * If more memory is needed, it is allocated from the heap. 226afc2ba1dSToomas Soome */ 227afc2ba1dSToomas Soome #define STACKLIMIT 9 228afc2ba1dSToomas Soome 229afc2ba1dSToomas Soome /* 230afc2ba1dSToomas Soome * Little Endian or Big Endian? 231afc2ba1dSToomas Soome * Note: overwrite the below #define if you know your architecture endianess. 232afc2ba1dSToomas Soome */ 233*10ae99eeSToomas Soome #if defined(BYTE_ORDER) 234*10ae99eeSToomas Soome #if BYTE_ORDER == BIG_ENDIAN /* This is sys/endian.h API */ 235*10ae99eeSToomas Soome #define LZ4_BIG_ENDIAN 1 236*10ae99eeSToomas Soome #else 237*10ae99eeSToomas Soome /* 238*10ae99eeSToomas Soome * Little Endian assumed. PDP Endian and other very rare endian format 239*10ae99eeSToomas Soome * are unsupported. 240*10ae99eeSToomas Soome */ 241*10ae99eeSToomas Soome #endif 242*10ae99eeSToomas Soome #else /* !defined(BYTE_ORDER) */ 243afc2ba1dSToomas Soome #if (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || \ 244afc2ba1dSToomas Soome defined(_BIG_ENDIAN) || defined(_ARCH_PPC) || defined(__PPC__) || \ 245afc2ba1dSToomas Soome defined(__PPC) || defined(PPC) || defined(__powerpc__) || \ 246afc2ba1dSToomas Soome defined(__powerpc) || defined(powerpc) || \ 247afc2ba1dSToomas Soome ((defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)))) 248afc2ba1dSToomas Soome #define LZ4_BIG_ENDIAN 1 249afc2ba1dSToomas Soome #else 250afc2ba1dSToomas Soome /* 251afc2ba1dSToomas Soome * Little Endian assumed. PDP Endian and other very rare endian format 252afc2ba1dSToomas Soome * are unsupported. 253afc2ba1dSToomas Soome */ 254afc2ba1dSToomas Soome #endif 255*10ae99eeSToomas Soome #endif /* defined(BYTE_ORDER) */ 256afc2ba1dSToomas Soome 257afc2ba1dSToomas Soome /* 258afc2ba1dSToomas Soome * Unaligned memory access is automatically enabled for "common" CPU, 259afc2ba1dSToomas Soome * such as x86. For others CPU, the compiler will be more cautious, and 260afc2ba1dSToomas Soome * insert extra code to ensure aligned access is respected. If you know 261afc2ba1dSToomas Soome * your target CPU supports unaligned memory access, you may want to 262afc2ba1dSToomas Soome * force this option manually to improve performance 263afc2ba1dSToomas Soome */ 264afc2ba1dSToomas Soome #if defined(__ARM_FEATURE_UNALIGNED) 265afc2ba1dSToomas Soome #define LZ4_FORCE_UNALIGNED_ACCESS 1 266afc2ba1dSToomas Soome #endif 267afc2ba1dSToomas Soome 268afc2ba1dSToomas Soome #ifdef __sparc 269afc2ba1dSToomas Soome #define LZ4_FORCE_SW_BITCOUNT 270afc2ba1dSToomas Soome #endif 271afc2ba1dSToomas Soome 272afc2ba1dSToomas Soome /* 273afc2ba1dSToomas Soome * Compiler Options 274afc2ba1dSToomas Soome */ 275afc2ba1dSToomas Soome #if __STDC_VERSION__ >= 199901L /* C99 */ 276afc2ba1dSToomas Soome /* "restrict" is a known keyword */ 277afc2ba1dSToomas Soome #else 278afc2ba1dSToomas Soome /* Disable restrict */ 279afc2ba1dSToomas Soome #define restrict 280afc2ba1dSToomas Soome #endif 281afc2ba1dSToomas Soome 282afc2ba1dSToomas Soome #define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 283afc2ba1dSToomas Soome 284afc2ba1dSToomas Soome #ifdef _MSC_VER 285afc2ba1dSToomas Soome /* Visual Studio */ 286afc2ba1dSToomas Soome /* Visual is not C99, but supports some kind of inline */ 287afc2ba1dSToomas Soome #define inline __forceinline 288afc2ba1dSToomas Soome #if LZ4_ARCH64 289afc2ba1dSToomas Soome /* For Visual 2005 */ 290afc2ba1dSToomas Soome #pragma intrinsic(_BitScanForward64) 291afc2ba1dSToomas Soome #pragma intrinsic(_BitScanReverse64) 292afc2ba1dSToomas Soome #else /* !LZ4_ARCH64 */ 293afc2ba1dSToomas Soome /* For Visual 2005 */ 294afc2ba1dSToomas Soome #pragma intrinsic(_BitScanForward) 295afc2ba1dSToomas Soome #pragma intrinsic(_BitScanReverse) 296afc2ba1dSToomas Soome #endif /* !LZ4_ARCH64 */ 297afc2ba1dSToomas Soome #endif /* _MSC_VER */ 298afc2ba1dSToomas Soome 299afc2ba1dSToomas Soome #ifdef _MSC_VER 300afc2ba1dSToomas Soome #define lz4_bswap16(x) _byteswap_ushort(x) 301afc2ba1dSToomas Soome #else /* !_MSC_VER */ 302afc2ba1dSToomas Soome #define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | \ 303afc2ba1dSToomas Soome (((x) & 0xffu) << 8))) 304afc2ba1dSToomas Soome #endif /* !_MSC_VER */ 305afc2ba1dSToomas Soome 306afc2ba1dSToomas Soome #if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) 307afc2ba1dSToomas Soome #define expect(expr, value) (__builtin_expect((expr), (value))) 308afc2ba1dSToomas Soome #else 309afc2ba1dSToomas Soome #define expect(expr, value) (expr) 310afc2ba1dSToomas Soome #endif 311afc2ba1dSToomas Soome 312*10ae99eeSToomas Soome #ifndef likely 313afc2ba1dSToomas Soome #define likely(expr) expect((expr) != 0, 1) 314*10ae99eeSToomas Soome #endif 315*10ae99eeSToomas Soome 316*10ae99eeSToomas Soome #ifndef unlikely 317afc2ba1dSToomas Soome #define unlikely(expr) expect((expr) != 0, 0) 318*10ae99eeSToomas Soome #endif 319afc2ba1dSToomas Soome 320afc2ba1dSToomas Soome /* Basic types */ 321afc2ba1dSToomas Soome #if defined(_MSC_VER) 322afc2ba1dSToomas Soome /* Visual Studio does not support 'stdint' natively */ 323afc2ba1dSToomas Soome #define BYTE unsigned __int8 324afc2ba1dSToomas Soome #define U16 unsigned __int16 325afc2ba1dSToomas Soome #define U32 unsigned __int32 326afc2ba1dSToomas Soome #define S32 __int32 327afc2ba1dSToomas Soome #define U64 unsigned __int64 328afc2ba1dSToomas Soome #else /* !defined(_MSC_VER) */ 329afc2ba1dSToomas Soome #define BYTE uint8_t 330afc2ba1dSToomas Soome #define U16 uint16_t 331afc2ba1dSToomas Soome #define U32 uint32_t 332afc2ba1dSToomas Soome #define S32 int32_t 333afc2ba1dSToomas Soome #define U64 uint64_t 334afc2ba1dSToomas Soome #endif /* !defined(_MSC_VER) */ 335afc2ba1dSToomas Soome 336afc2ba1dSToomas Soome #ifndef LZ4_FORCE_UNALIGNED_ACCESS 337afc2ba1dSToomas Soome #pragma pack(1) 338afc2ba1dSToomas Soome #endif 339afc2ba1dSToomas Soome 340afc2ba1dSToomas Soome typedef struct _U16_S { 341afc2ba1dSToomas Soome U16 v; 342afc2ba1dSToomas Soome } U16_S; 343afc2ba1dSToomas Soome typedef struct _U32_S { 344afc2ba1dSToomas Soome U32 v; 345afc2ba1dSToomas Soome } U32_S; 346afc2ba1dSToomas Soome typedef struct _U64_S { 347afc2ba1dSToomas Soome U64 v; 348afc2ba1dSToomas Soome } U64_S; 349afc2ba1dSToomas Soome 350afc2ba1dSToomas Soome #ifndef LZ4_FORCE_UNALIGNED_ACCESS 351afc2ba1dSToomas Soome #pragma pack() 352afc2ba1dSToomas Soome #endif 353afc2ba1dSToomas Soome 354*10ae99eeSToomas Soome #define A64(x) (((U64_S *)(__DECONST(void *, x)))->v) 355*10ae99eeSToomas Soome #define A32(x) (((U32_S *)(__DECONST(void *, x)))->v) 356*10ae99eeSToomas Soome #define A16(x) (((U16_S *)(__DECONST(void *, x)))->v) 357afc2ba1dSToomas Soome 358afc2ba1dSToomas Soome /* 359afc2ba1dSToomas Soome * Constants 360afc2ba1dSToomas Soome */ 361afc2ba1dSToomas Soome #define MINMATCH 4 362afc2ba1dSToomas Soome 363afc2ba1dSToomas Soome #define HASH_LOG COMPRESSIONLEVEL 364afc2ba1dSToomas Soome #define HASHTABLESIZE (1 << HASH_LOG) 365afc2ba1dSToomas Soome #define HASH_MASK (HASHTABLESIZE - 1) 366afc2ba1dSToomas Soome 367afc2ba1dSToomas Soome #define SKIPSTRENGTH (NOTCOMPRESSIBLE_CONFIRMATION > 2 ? \ 368afc2ba1dSToomas Soome NOTCOMPRESSIBLE_CONFIRMATION : 2) 369afc2ba1dSToomas Soome 370afc2ba1dSToomas Soome /* 371afc2ba1dSToomas Soome * Defines if memory is allocated into the stack (local variable), 372afc2ba1dSToomas Soome * or into the heap (kmem_alloc()). 373afc2ba1dSToomas Soome */ 374afc2ba1dSToomas Soome #define HEAPMODE (HASH_LOG > STACKLIMIT) 375afc2ba1dSToomas Soome #define COPYLENGTH 8 376afc2ba1dSToomas Soome #define LASTLITERALS 5 377afc2ba1dSToomas Soome #define MFLIMIT (COPYLENGTH + MINMATCH) 378afc2ba1dSToomas Soome #define MINLENGTH (MFLIMIT + 1) 379afc2ba1dSToomas Soome 380afc2ba1dSToomas Soome #define MAXD_LOG 16 381afc2ba1dSToomas Soome #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) 382afc2ba1dSToomas Soome 383afc2ba1dSToomas Soome #define ML_BITS 4 384afc2ba1dSToomas Soome #define ML_MASK ((1U<<ML_BITS)-1) 385afc2ba1dSToomas Soome #define RUN_BITS (8-ML_BITS) 386afc2ba1dSToomas Soome #define RUN_MASK ((1U<<RUN_BITS)-1) 387afc2ba1dSToomas Soome 388afc2ba1dSToomas Soome 389afc2ba1dSToomas Soome /* 390afc2ba1dSToomas Soome * Architecture-specific macros 391afc2ba1dSToomas Soome */ 392afc2ba1dSToomas Soome #if LZ4_ARCH64 393afc2ba1dSToomas Soome #define STEPSIZE 8 394afc2ba1dSToomas Soome #define UARCH U64 395afc2ba1dSToomas Soome #define AARCH A64 396afc2ba1dSToomas Soome #define LZ4_COPYSTEP(s, d) A64(d) = A64(s); d += 8; s += 8; 397afc2ba1dSToomas Soome #define LZ4_COPYPACKET(s, d) LZ4_COPYSTEP(s, d) 398afc2ba1dSToomas Soome #define LZ4_SECURECOPY(s, d, e) if (d < e) LZ4_WILDCOPY(s, d, e) 399afc2ba1dSToomas Soome #define HTYPE U32 400afc2ba1dSToomas Soome #define INITBASE(base) const BYTE* const base = ip 401afc2ba1dSToomas Soome #else /* !LZ4_ARCH64 */ 402afc2ba1dSToomas Soome #define STEPSIZE 4 403afc2ba1dSToomas Soome #define UARCH U32 404afc2ba1dSToomas Soome #define AARCH A32 405afc2ba1dSToomas Soome #define LZ4_COPYSTEP(s, d) A32(d) = A32(s); d += 4; s += 4; 406afc2ba1dSToomas Soome #define LZ4_COPYPACKET(s, d) LZ4_COPYSTEP(s, d); LZ4_COPYSTEP(s, d); 407afc2ba1dSToomas Soome #define LZ4_SECURECOPY LZ4_WILDCOPY 408afc2ba1dSToomas Soome #define HTYPE const BYTE * 409afc2ba1dSToomas Soome #define INITBASE(base) const int base = 0 410afc2ba1dSToomas Soome #endif /* !LZ4_ARCH64 */ 411afc2ba1dSToomas Soome 412afc2ba1dSToomas Soome #if (defined(LZ4_BIG_ENDIAN) && !defined(BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE)) 413afc2ba1dSToomas Soome #define LZ4_READ_LITTLEENDIAN_16(d, s, p) \ 414afc2ba1dSToomas Soome { U16 v = A16(p); v = lz4_bswap16(v); d = (s) - v; } 415afc2ba1dSToomas Soome #define LZ4_WRITE_LITTLEENDIAN_16(p, i) \ 416afc2ba1dSToomas Soome { U16 v = (U16)(i); v = lz4_bswap16(v); A16(p) = v; p += 2; } 417afc2ba1dSToomas Soome #else 418afc2ba1dSToomas Soome #define LZ4_READ_LITTLEENDIAN_16(d, s, p) { d = (s) - A16(p); } 419afc2ba1dSToomas Soome #define LZ4_WRITE_LITTLEENDIAN_16(p, v) { A16(p) = v; p += 2; } 420afc2ba1dSToomas Soome #endif 421afc2ba1dSToomas Soome 422afc2ba1dSToomas Soome 423afc2ba1dSToomas Soome /* Local structures */ 424afc2ba1dSToomas Soome struct refTables { 425afc2ba1dSToomas Soome HTYPE hashTable[HASHTABLESIZE]; 426afc2ba1dSToomas Soome }; 427afc2ba1dSToomas Soome 428afc2ba1dSToomas Soome 429afc2ba1dSToomas Soome /* Macros */ 430afc2ba1dSToomas Soome #define LZ4_HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH * 8) - \ 431afc2ba1dSToomas Soome HASH_LOG)) 432afc2ba1dSToomas Soome #define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p)) 433afc2ba1dSToomas Soome #define LZ4_WILDCOPY(s, d, e) do { LZ4_COPYPACKET(s, d) } while (d < e); 434afc2ba1dSToomas Soome #define LZ4_BLINDCOPY(s, d, l) { BYTE* e = (d) + l; LZ4_WILDCOPY(s, d, e); \ 435afc2ba1dSToomas Soome d = e; } 436afc2ba1dSToomas Soome 437afc2ba1dSToomas Soome 438afc2ba1dSToomas Soome /* Private functions */ 439afc2ba1dSToomas Soome #if LZ4_ARCH64 440afc2ba1dSToomas Soome 441*10ae99eeSToomas Soome static inline int 442afc2ba1dSToomas Soome LZ4_NbCommonBytes(register U64 val) 443afc2ba1dSToomas Soome { 444afc2ba1dSToomas Soome #if defined(LZ4_BIG_ENDIAN) 445afc2ba1dSToomas Soome #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 446afc2ba1dSToomas Soome unsigned long r = 0; 447afc2ba1dSToomas Soome _BitScanReverse64(&r, val); 448afc2ba1dSToomas Soome return (int)(r >> 3); 449afc2ba1dSToomas Soome #elif defined(__GNUC__) && (GCC_VERSION >= 304) && \ 450afc2ba1dSToomas Soome !defined(LZ4_FORCE_SW_BITCOUNT) 451afc2ba1dSToomas Soome return (__builtin_clzll(val) >> 3); 452afc2ba1dSToomas Soome #else 453afc2ba1dSToomas Soome int r; 454afc2ba1dSToomas Soome if (!(val >> 32)) { 455afc2ba1dSToomas Soome r = 4; 456afc2ba1dSToomas Soome } else { 457afc2ba1dSToomas Soome r = 0; 458afc2ba1dSToomas Soome val >>= 32; 459afc2ba1dSToomas Soome } 460afc2ba1dSToomas Soome if (!(val >> 16)) { 461afc2ba1dSToomas Soome r += 2; 462afc2ba1dSToomas Soome val >>= 8; 463afc2ba1dSToomas Soome } else { 464afc2ba1dSToomas Soome val >>= 24; 465afc2ba1dSToomas Soome } 466afc2ba1dSToomas Soome r += (!val); 467afc2ba1dSToomas Soome return (r); 468afc2ba1dSToomas Soome #endif 469afc2ba1dSToomas Soome #else 470afc2ba1dSToomas Soome #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 471afc2ba1dSToomas Soome unsigned long r = 0; 472afc2ba1dSToomas Soome _BitScanForward64(&r, val); 473afc2ba1dSToomas Soome return (int)(r >> 3); 474afc2ba1dSToomas Soome #elif defined(__GNUC__) && (GCC_VERSION >= 304) && \ 475afc2ba1dSToomas Soome !defined(LZ4_FORCE_SW_BITCOUNT) 476afc2ba1dSToomas Soome return (__builtin_ctzll(val) >> 3); 477afc2ba1dSToomas Soome #else 478afc2ba1dSToomas Soome static const int DeBruijnBytePos[64] = 479afc2ba1dSToomas Soome { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 480afc2ba1dSToomas Soome 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 481afc2ba1dSToomas Soome 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 482afc2ba1dSToomas Soome 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 483afc2ba1dSToomas Soome }; 484afc2ba1dSToomas Soome return DeBruijnBytePos[((U64) ((val & -val) * 0x0218A392CDABBD3F)) >> 485afc2ba1dSToomas Soome 58]; 486afc2ba1dSToomas Soome #endif 487afc2ba1dSToomas Soome #endif 488afc2ba1dSToomas Soome } 489afc2ba1dSToomas Soome 490afc2ba1dSToomas Soome #else 491afc2ba1dSToomas Soome 492*10ae99eeSToomas Soome static inline int 493afc2ba1dSToomas Soome LZ4_NbCommonBytes(register U32 val) 494afc2ba1dSToomas Soome { 495afc2ba1dSToomas Soome #if defined(LZ4_BIG_ENDIAN) 496afc2ba1dSToomas Soome #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 497afc2ba1dSToomas Soome unsigned long r = 0; 498afc2ba1dSToomas Soome _BitScanReverse(&r, val); 499afc2ba1dSToomas Soome return (int)(r >> 3); 500afc2ba1dSToomas Soome #elif defined(__GNUC__) && (GCC_VERSION >= 304) && \ 501afc2ba1dSToomas Soome !defined(LZ4_FORCE_SW_BITCOUNT) 502afc2ba1dSToomas Soome return (__builtin_clz(val) >> 3); 503afc2ba1dSToomas Soome #else 504afc2ba1dSToomas Soome int r; 505afc2ba1dSToomas Soome if (!(val >> 16)) { 506afc2ba1dSToomas Soome r = 2; 507afc2ba1dSToomas Soome val >>= 8; 508afc2ba1dSToomas Soome } else { 509afc2ba1dSToomas Soome r = 0; 510afc2ba1dSToomas Soome val >>= 24; 511afc2ba1dSToomas Soome } 512afc2ba1dSToomas Soome r += (!val); 513afc2ba1dSToomas Soome return (r); 514afc2ba1dSToomas Soome #endif 515afc2ba1dSToomas Soome #else 516afc2ba1dSToomas Soome #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 517afc2ba1dSToomas Soome unsigned long r = 0; 518afc2ba1dSToomas Soome _BitScanForward(&r, val); 519afc2ba1dSToomas Soome return (int)(r >> 3); 520afc2ba1dSToomas Soome #elif defined(__GNUC__) && (GCC_VERSION >= 304) && \ 521afc2ba1dSToomas Soome !defined(LZ4_FORCE_SW_BITCOUNT) 522afc2ba1dSToomas Soome return (__builtin_ctz(val) >> 3); 523afc2ba1dSToomas Soome #else 524afc2ba1dSToomas Soome static const int DeBruijnBytePos[32] = { 525afc2ba1dSToomas Soome 0, 0, 3, 0, 3, 1, 3, 0, 526afc2ba1dSToomas Soome 3, 2, 2, 1, 3, 2, 0, 1, 527afc2ba1dSToomas Soome 3, 3, 1, 2, 2, 2, 2, 0, 528afc2ba1dSToomas Soome 3, 1, 2, 0, 1, 0, 1, 1 529afc2ba1dSToomas Soome }; 530afc2ba1dSToomas Soome return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 531afc2ba1dSToomas Soome 27]; 532afc2ba1dSToomas Soome #endif 533afc2ba1dSToomas Soome #endif 534afc2ba1dSToomas Soome } 535afc2ba1dSToomas Soome 536afc2ba1dSToomas Soome #endif 537afc2ba1dSToomas Soome 538afc2ba1dSToomas Soome /* Compression functions */ 539afc2ba1dSToomas Soome 540afc2ba1dSToomas Soome /*ARGSUSED*/ 541afc2ba1dSToomas Soome static int 542afc2ba1dSToomas Soome LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize, 543afc2ba1dSToomas Soome int osize) 544afc2ba1dSToomas Soome { 545afc2ba1dSToomas Soome #if HEAPMODE 546afc2ba1dSToomas Soome struct refTables *srt = (struct refTables *)ctx; 547afc2ba1dSToomas Soome HTYPE *HashTable = (HTYPE *) (srt->hashTable); 548afc2ba1dSToomas Soome #else 549afc2ba1dSToomas Soome HTYPE HashTable[HASHTABLESIZE] = { 0 }; 550afc2ba1dSToomas Soome #endif 551afc2ba1dSToomas Soome 552*10ae99eeSToomas Soome const BYTE *ip = (const BYTE *) source; 553afc2ba1dSToomas Soome INITBASE(base); 554afc2ba1dSToomas Soome const BYTE *anchor = ip; 555afc2ba1dSToomas Soome const BYTE *const iend = ip + isize; 556afc2ba1dSToomas Soome const BYTE *const oend = (BYTE *) dest + osize; 557afc2ba1dSToomas Soome const BYTE *const mflimit = iend - MFLIMIT; 558afc2ba1dSToomas Soome #define matchlimit (iend - LASTLITERALS) 559afc2ba1dSToomas Soome 560afc2ba1dSToomas Soome BYTE *op = (BYTE *) dest; 561afc2ba1dSToomas Soome 562afc2ba1dSToomas Soome int len, length; 563afc2ba1dSToomas Soome const int skipStrength = SKIPSTRENGTH; 564afc2ba1dSToomas Soome U32 forwardH; 565afc2ba1dSToomas Soome 566afc2ba1dSToomas Soome 567afc2ba1dSToomas Soome /* Init */ 568afc2ba1dSToomas Soome if (isize < MINLENGTH) 569afc2ba1dSToomas Soome goto _last_literals; 570afc2ba1dSToomas Soome 571afc2ba1dSToomas Soome /* First Byte */ 572afc2ba1dSToomas Soome HashTable[LZ4_HASH_VALUE(ip)] = ip - base; 573afc2ba1dSToomas Soome ip++; 574afc2ba1dSToomas Soome forwardH = LZ4_HASH_VALUE(ip); 575afc2ba1dSToomas Soome 576afc2ba1dSToomas Soome /* Main Loop */ 577afc2ba1dSToomas Soome for (;;) { 578afc2ba1dSToomas Soome int findMatchAttempts = (1U << skipStrength) + 3; 579afc2ba1dSToomas Soome const BYTE *forwardIp = ip; 580afc2ba1dSToomas Soome const BYTE *ref; 581afc2ba1dSToomas Soome BYTE *token; 582afc2ba1dSToomas Soome 583afc2ba1dSToomas Soome /* Find a match */ 584afc2ba1dSToomas Soome do { 585afc2ba1dSToomas Soome U32 h = forwardH; 586afc2ba1dSToomas Soome int step = findMatchAttempts++ >> skipStrength; 587afc2ba1dSToomas Soome ip = forwardIp; 588afc2ba1dSToomas Soome forwardIp = ip + step; 589afc2ba1dSToomas Soome 590afc2ba1dSToomas Soome if unlikely(forwardIp > mflimit) { 591afc2ba1dSToomas Soome goto _last_literals; 592afc2ba1dSToomas Soome } 593afc2ba1dSToomas Soome 594afc2ba1dSToomas Soome forwardH = LZ4_HASH_VALUE(forwardIp); 595afc2ba1dSToomas Soome ref = base + HashTable[h]; 596afc2ba1dSToomas Soome HashTable[h] = ip - base; 597afc2ba1dSToomas Soome 598afc2ba1dSToomas Soome } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); 599afc2ba1dSToomas Soome 600afc2ba1dSToomas Soome /* Catch up */ 601*10ae99eeSToomas Soome while ((ip > anchor) && (ref > (const BYTE *) source) && 602afc2ba1dSToomas Soome unlikely(ip[-1] == ref[-1])) { 603afc2ba1dSToomas Soome ip--; 604afc2ba1dSToomas Soome ref--; 605afc2ba1dSToomas Soome } 606afc2ba1dSToomas Soome 607afc2ba1dSToomas Soome /* Encode Literal length */ 608afc2ba1dSToomas Soome length = ip - anchor; 609afc2ba1dSToomas Soome token = op++; 610afc2ba1dSToomas Soome 611afc2ba1dSToomas Soome /* Check output limit */ 612afc2ba1dSToomas Soome if unlikely(op + length + (2 + 1 + LASTLITERALS) + 613afc2ba1dSToomas Soome (length >> 8) > oend) 614afc2ba1dSToomas Soome return (0); 615afc2ba1dSToomas Soome 616afc2ba1dSToomas Soome if (length >= (int)RUN_MASK) { 617afc2ba1dSToomas Soome *token = (RUN_MASK << ML_BITS); 618afc2ba1dSToomas Soome len = length - RUN_MASK; 619afc2ba1dSToomas Soome for (; len > 254; len -= 255) 620afc2ba1dSToomas Soome *op++ = 255; 621afc2ba1dSToomas Soome *op++ = (BYTE)len; 622afc2ba1dSToomas Soome } else 623afc2ba1dSToomas Soome *token = (length << ML_BITS); 624afc2ba1dSToomas Soome 625afc2ba1dSToomas Soome /* Copy Literals */ 626afc2ba1dSToomas Soome LZ4_BLINDCOPY(anchor, op, length); 627afc2ba1dSToomas Soome 628afc2ba1dSToomas Soome _next_match: 629afc2ba1dSToomas Soome /* Encode Offset */ 630afc2ba1dSToomas Soome LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref); 631afc2ba1dSToomas Soome 632afc2ba1dSToomas Soome /* Start Counting */ 633afc2ba1dSToomas Soome ip += MINMATCH; 634afc2ba1dSToomas Soome ref += MINMATCH; /* MinMatch verified */ 635afc2ba1dSToomas Soome anchor = ip; 636afc2ba1dSToomas Soome while likely(ip < matchlimit - (STEPSIZE - 1)) { 637afc2ba1dSToomas Soome UARCH diff = AARCH(ref) ^ AARCH(ip); 638afc2ba1dSToomas Soome if (!diff) { 639afc2ba1dSToomas Soome ip += STEPSIZE; 640afc2ba1dSToomas Soome ref += STEPSIZE; 641afc2ba1dSToomas Soome continue; 642afc2ba1dSToomas Soome } 643afc2ba1dSToomas Soome ip += LZ4_NbCommonBytes(diff); 644afc2ba1dSToomas Soome goto _endCount; 645afc2ba1dSToomas Soome } 646afc2ba1dSToomas Soome #if LZ4_ARCH64 647afc2ba1dSToomas Soome if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) { 648afc2ba1dSToomas Soome ip += 4; 649afc2ba1dSToomas Soome ref += 4; 650afc2ba1dSToomas Soome } 651afc2ba1dSToomas Soome #endif 652afc2ba1dSToomas Soome if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) { 653afc2ba1dSToomas Soome ip += 2; 654afc2ba1dSToomas Soome ref += 2; 655afc2ba1dSToomas Soome } 656afc2ba1dSToomas Soome if ((ip < matchlimit) && (*ref == *ip)) 657afc2ba1dSToomas Soome ip++; 658afc2ba1dSToomas Soome _endCount: 659afc2ba1dSToomas Soome 660afc2ba1dSToomas Soome /* Encode MatchLength */ 661afc2ba1dSToomas Soome len = (ip - anchor); 662afc2ba1dSToomas Soome /* Check output limit */ 663afc2ba1dSToomas Soome if unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend) 664afc2ba1dSToomas Soome return (0); 665afc2ba1dSToomas Soome if (len >= (int)ML_MASK) { 666afc2ba1dSToomas Soome *token += ML_MASK; 667afc2ba1dSToomas Soome len -= ML_MASK; 668afc2ba1dSToomas Soome for (; len > 509; len -= 510) { 669afc2ba1dSToomas Soome *op++ = 255; 670afc2ba1dSToomas Soome *op++ = 255; 671afc2ba1dSToomas Soome } 672afc2ba1dSToomas Soome if (len > 254) { 673afc2ba1dSToomas Soome len -= 255; 674afc2ba1dSToomas Soome *op++ = 255; 675afc2ba1dSToomas Soome } 676afc2ba1dSToomas Soome *op++ = (BYTE)len; 677afc2ba1dSToomas Soome } else 678afc2ba1dSToomas Soome *token += len; 679afc2ba1dSToomas Soome 680afc2ba1dSToomas Soome /* Test end of chunk */ 681afc2ba1dSToomas Soome if (ip > mflimit) { 682afc2ba1dSToomas Soome anchor = ip; 683afc2ba1dSToomas Soome break; 684afc2ba1dSToomas Soome } 685afc2ba1dSToomas Soome /* Fill table */ 686afc2ba1dSToomas Soome HashTable[LZ4_HASH_VALUE(ip - 2)] = ip - 2 - base; 687afc2ba1dSToomas Soome 688afc2ba1dSToomas Soome /* Test next position */ 689afc2ba1dSToomas Soome ref = base + HashTable[LZ4_HASH_VALUE(ip)]; 690afc2ba1dSToomas Soome HashTable[LZ4_HASH_VALUE(ip)] = ip - base; 691afc2ba1dSToomas Soome if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { 692afc2ba1dSToomas Soome token = op++; 693afc2ba1dSToomas Soome *token = 0; 694afc2ba1dSToomas Soome goto _next_match; 695afc2ba1dSToomas Soome } 696afc2ba1dSToomas Soome /* Prepare next loop */ 697afc2ba1dSToomas Soome anchor = ip++; 698afc2ba1dSToomas Soome forwardH = LZ4_HASH_VALUE(ip); 699afc2ba1dSToomas Soome } 700afc2ba1dSToomas Soome 701afc2ba1dSToomas Soome _last_literals: 702afc2ba1dSToomas Soome /* Encode Last Literals */ 703afc2ba1dSToomas Soome { 704afc2ba1dSToomas Soome int lastRun = iend - anchor; 705afc2ba1dSToomas Soome if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) > 706afc2ba1dSToomas Soome oend) 707afc2ba1dSToomas Soome return (0); 708afc2ba1dSToomas Soome if (lastRun >= (int)RUN_MASK) { 709afc2ba1dSToomas Soome *op++ = (RUN_MASK << ML_BITS); 710afc2ba1dSToomas Soome lastRun -= RUN_MASK; 711afc2ba1dSToomas Soome for (; lastRun > 254; lastRun -= 255) { 712afc2ba1dSToomas Soome *op++ = 255; 713afc2ba1dSToomas Soome } 714afc2ba1dSToomas Soome *op++ = (BYTE)lastRun; 715afc2ba1dSToomas Soome } else 716afc2ba1dSToomas Soome *op++ = (lastRun << ML_BITS); 717afc2ba1dSToomas Soome (void) memcpy(op, anchor, iend - anchor); 718afc2ba1dSToomas Soome op += iend - anchor; 719afc2ba1dSToomas Soome } 720afc2ba1dSToomas Soome 721afc2ba1dSToomas Soome /* End */ 722afc2ba1dSToomas Soome return (int)(((char *)op) - dest); 723afc2ba1dSToomas Soome } 724afc2ba1dSToomas Soome 725afc2ba1dSToomas Soome 726afc2ba1dSToomas Soome 727afc2ba1dSToomas Soome /* Note : this function is valid only if isize < LZ4_64KLIMIT */ 728afc2ba1dSToomas Soome #define LZ4_64KLIMIT ((1 << 16) + (MFLIMIT - 1)) 729afc2ba1dSToomas Soome #define HASHLOG64K (HASH_LOG + 1) 730afc2ba1dSToomas Soome #define HASH64KTABLESIZE (1U << HASHLOG64K) 731afc2ba1dSToomas Soome #define LZ4_HASH64K_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8) - \ 732afc2ba1dSToomas Soome HASHLOG64K)) 733afc2ba1dSToomas Soome #define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p)) 734afc2ba1dSToomas Soome 735afc2ba1dSToomas Soome /*ARGSUSED*/ 736afc2ba1dSToomas Soome static int 737afc2ba1dSToomas Soome LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize, 738afc2ba1dSToomas Soome int osize) 739afc2ba1dSToomas Soome { 740afc2ba1dSToomas Soome #if HEAPMODE 741afc2ba1dSToomas Soome struct refTables *srt = (struct refTables *)ctx; 742afc2ba1dSToomas Soome U16 *HashTable = (U16 *) (srt->hashTable); 743afc2ba1dSToomas Soome #else 744afc2ba1dSToomas Soome U16 HashTable[HASH64KTABLESIZE] = { 0 }; 745afc2ba1dSToomas Soome #endif 746afc2ba1dSToomas Soome 747*10ae99eeSToomas Soome const BYTE *ip = (const BYTE *) source; 748afc2ba1dSToomas Soome const BYTE *anchor = ip; 749afc2ba1dSToomas Soome const BYTE *const base = ip; 750afc2ba1dSToomas Soome const BYTE *const iend = ip + isize; 751afc2ba1dSToomas Soome const BYTE *const oend = (BYTE *) dest + osize; 752afc2ba1dSToomas Soome const BYTE *const mflimit = iend - MFLIMIT; 753afc2ba1dSToomas Soome #define matchlimit (iend - LASTLITERALS) 754afc2ba1dSToomas Soome 755afc2ba1dSToomas Soome BYTE *op = (BYTE *) dest; 756afc2ba1dSToomas Soome 757afc2ba1dSToomas Soome int len, length; 758afc2ba1dSToomas Soome const int skipStrength = SKIPSTRENGTH; 759afc2ba1dSToomas Soome U32 forwardH; 760afc2ba1dSToomas Soome 761afc2ba1dSToomas Soome /* Init */ 762afc2ba1dSToomas Soome if (isize < MINLENGTH) 763afc2ba1dSToomas Soome goto _last_literals; 764afc2ba1dSToomas Soome 765afc2ba1dSToomas Soome /* First Byte */ 766afc2ba1dSToomas Soome ip++; 767afc2ba1dSToomas Soome forwardH = LZ4_HASH64K_VALUE(ip); 768afc2ba1dSToomas Soome 769afc2ba1dSToomas Soome /* Main Loop */ 770afc2ba1dSToomas Soome for (;;) { 771afc2ba1dSToomas Soome int findMatchAttempts = (1U << skipStrength) + 3; 772afc2ba1dSToomas Soome const BYTE *forwardIp = ip; 773afc2ba1dSToomas Soome const BYTE *ref; 774afc2ba1dSToomas Soome BYTE *token; 775afc2ba1dSToomas Soome 776afc2ba1dSToomas Soome /* Find a match */ 777afc2ba1dSToomas Soome do { 778afc2ba1dSToomas Soome U32 h = forwardH; 779afc2ba1dSToomas Soome int step = findMatchAttempts++ >> skipStrength; 780afc2ba1dSToomas Soome ip = forwardIp; 781afc2ba1dSToomas Soome forwardIp = ip + step; 782afc2ba1dSToomas Soome 783afc2ba1dSToomas Soome if (forwardIp > mflimit) { 784afc2ba1dSToomas Soome goto _last_literals; 785afc2ba1dSToomas Soome } 786afc2ba1dSToomas Soome 787afc2ba1dSToomas Soome forwardH = LZ4_HASH64K_VALUE(forwardIp); 788afc2ba1dSToomas Soome ref = base + HashTable[h]; 789afc2ba1dSToomas Soome HashTable[h] = ip - base; 790afc2ba1dSToomas Soome 791afc2ba1dSToomas Soome } while (A32(ref) != A32(ip)); 792afc2ba1dSToomas Soome 793afc2ba1dSToomas Soome /* Catch up */ 794*10ae99eeSToomas Soome while ((ip > anchor) && (ref > (const BYTE *) source) && 795afc2ba1dSToomas Soome (ip[-1] == ref[-1])) { 796afc2ba1dSToomas Soome ip--; 797afc2ba1dSToomas Soome ref--; 798afc2ba1dSToomas Soome } 799afc2ba1dSToomas Soome 800afc2ba1dSToomas Soome /* Encode Literal length */ 801afc2ba1dSToomas Soome length = ip - anchor; 802afc2ba1dSToomas Soome token = op++; 803afc2ba1dSToomas Soome 804afc2ba1dSToomas Soome /* Check output limit */ 805afc2ba1dSToomas Soome if unlikely(op + length + (2 + 1 + LASTLITERALS) + 806afc2ba1dSToomas Soome (length >> 8) > oend) 807afc2ba1dSToomas Soome return (0); 808afc2ba1dSToomas Soome 809afc2ba1dSToomas Soome if (length >= (int)RUN_MASK) { 810afc2ba1dSToomas Soome *token = (RUN_MASK << ML_BITS); 811afc2ba1dSToomas Soome len = length - RUN_MASK; 812afc2ba1dSToomas Soome for (; len > 254; len -= 255) 813afc2ba1dSToomas Soome *op++ = 255; 814afc2ba1dSToomas Soome *op++ = (BYTE)len; 815afc2ba1dSToomas Soome } else 816afc2ba1dSToomas Soome *token = (length << ML_BITS); 817afc2ba1dSToomas Soome 818afc2ba1dSToomas Soome /* Copy Literals */ 819afc2ba1dSToomas Soome LZ4_BLINDCOPY(anchor, op, length); 820afc2ba1dSToomas Soome 821afc2ba1dSToomas Soome _next_match: 822afc2ba1dSToomas Soome /* Encode Offset */ 823afc2ba1dSToomas Soome LZ4_WRITE_LITTLEENDIAN_16(op, ip - ref); 824afc2ba1dSToomas Soome 825afc2ba1dSToomas Soome /* Start Counting */ 826afc2ba1dSToomas Soome ip += MINMATCH; 827afc2ba1dSToomas Soome ref += MINMATCH; /* MinMatch verified */ 828afc2ba1dSToomas Soome anchor = ip; 829afc2ba1dSToomas Soome while (ip < matchlimit - (STEPSIZE - 1)) { 830afc2ba1dSToomas Soome UARCH diff = AARCH(ref) ^ AARCH(ip); 831afc2ba1dSToomas Soome if (!diff) { 832afc2ba1dSToomas Soome ip += STEPSIZE; 833afc2ba1dSToomas Soome ref += STEPSIZE; 834afc2ba1dSToomas Soome continue; 835afc2ba1dSToomas Soome } 836afc2ba1dSToomas Soome ip += LZ4_NbCommonBytes(diff); 837afc2ba1dSToomas Soome goto _endCount; 838afc2ba1dSToomas Soome } 839afc2ba1dSToomas Soome #if LZ4_ARCH64 840afc2ba1dSToomas Soome if ((ip < (matchlimit - 3)) && (A32(ref) == A32(ip))) { 841afc2ba1dSToomas Soome ip += 4; 842afc2ba1dSToomas Soome ref += 4; 843afc2ba1dSToomas Soome } 844afc2ba1dSToomas Soome #endif 845afc2ba1dSToomas Soome if ((ip < (matchlimit - 1)) && (A16(ref) == A16(ip))) { 846afc2ba1dSToomas Soome ip += 2; 847afc2ba1dSToomas Soome ref += 2; 848afc2ba1dSToomas Soome } 849afc2ba1dSToomas Soome if ((ip < matchlimit) && (*ref == *ip)) 850afc2ba1dSToomas Soome ip++; 851afc2ba1dSToomas Soome _endCount: 852afc2ba1dSToomas Soome 853afc2ba1dSToomas Soome /* Encode MatchLength */ 854afc2ba1dSToomas Soome len = (ip - anchor); 855afc2ba1dSToomas Soome /* Check output limit */ 856afc2ba1dSToomas Soome if unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend) 857afc2ba1dSToomas Soome return (0); 858afc2ba1dSToomas Soome if (len >= (int)ML_MASK) { 859afc2ba1dSToomas Soome *token += ML_MASK; 860afc2ba1dSToomas Soome len -= ML_MASK; 861afc2ba1dSToomas Soome for (; len > 509; len -= 510) { 862afc2ba1dSToomas Soome *op++ = 255; 863afc2ba1dSToomas Soome *op++ = 255; 864afc2ba1dSToomas Soome } 865afc2ba1dSToomas Soome if (len > 254) { 866afc2ba1dSToomas Soome len -= 255; 867afc2ba1dSToomas Soome *op++ = 255; 868afc2ba1dSToomas Soome } 869afc2ba1dSToomas Soome *op++ = (BYTE)len; 870afc2ba1dSToomas Soome } else 871afc2ba1dSToomas Soome *token += len; 872afc2ba1dSToomas Soome 873afc2ba1dSToomas Soome /* Test end of chunk */ 874afc2ba1dSToomas Soome if (ip > mflimit) { 875afc2ba1dSToomas Soome anchor = ip; 876afc2ba1dSToomas Soome break; 877afc2ba1dSToomas Soome } 878afc2ba1dSToomas Soome /* Fill table */ 879afc2ba1dSToomas Soome HashTable[LZ4_HASH64K_VALUE(ip - 2)] = ip - 2 - base; 880afc2ba1dSToomas Soome 881afc2ba1dSToomas Soome /* Test next position */ 882afc2ba1dSToomas Soome ref = base + HashTable[LZ4_HASH64K_VALUE(ip)]; 883afc2ba1dSToomas Soome HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base; 884afc2ba1dSToomas Soome if (A32(ref) == A32(ip)) { 885afc2ba1dSToomas Soome token = op++; 886afc2ba1dSToomas Soome *token = 0; 887afc2ba1dSToomas Soome goto _next_match; 888afc2ba1dSToomas Soome } 889afc2ba1dSToomas Soome /* Prepare next loop */ 890afc2ba1dSToomas Soome anchor = ip++; 891afc2ba1dSToomas Soome forwardH = LZ4_HASH64K_VALUE(ip); 892afc2ba1dSToomas Soome } 893afc2ba1dSToomas Soome 894afc2ba1dSToomas Soome _last_literals: 895afc2ba1dSToomas Soome /* Encode Last Literals */ 896afc2ba1dSToomas Soome { 897afc2ba1dSToomas Soome int lastRun = iend - anchor; 898afc2ba1dSToomas Soome if (op + lastRun + 1 + ((lastRun + 255 - RUN_MASK) / 255) > 899afc2ba1dSToomas Soome oend) 900afc2ba1dSToomas Soome return (0); 901afc2ba1dSToomas Soome if (lastRun >= (int)RUN_MASK) { 902afc2ba1dSToomas Soome *op++ = (RUN_MASK << ML_BITS); 903afc2ba1dSToomas Soome lastRun -= RUN_MASK; 904afc2ba1dSToomas Soome for (; lastRun > 254; lastRun -= 255) 905afc2ba1dSToomas Soome *op++ = 255; 906afc2ba1dSToomas Soome *op++ = (BYTE)lastRun; 907afc2ba1dSToomas Soome } else 908afc2ba1dSToomas Soome *op++ = (lastRun << ML_BITS); 909afc2ba1dSToomas Soome (void) memcpy(op, anchor, iend - anchor); 910afc2ba1dSToomas Soome op += iend - anchor; 911afc2ba1dSToomas Soome } 912afc2ba1dSToomas Soome 913afc2ba1dSToomas Soome /* End */ 914afc2ba1dSToomas Soome return (int)(((char *)op) - dest); 915afc2ba1dSToomas Soome } 916afc2ba1dSToomas Soome 917afc2ba1dSToomas Soome static int 918afc2ba1dSToomas Soome real_LZ4_compress(const char *source, char *dest, int isize, int osize) 919afc2ba1dSToomas Soome { 920afc2ba1dSToomas Soome #if HEAPMODE 921*10ae99eeSToomas Soome #if defined(_KERNEL) || defined(_FAKE_KERNEL) 922*10ae99eeSToomas Soome void *ctx = kmem_zalloc(sizeof (struct refTables), KM_NOSLEEP); 923*10ae99eeSToomas Soome #else 924*10ae99eeSToomas Soome void *ctx = malloc(sizeof (struct refTables)); 925*10ae99eeSToomas Soome #endif 926afc2ba1dSToomas Soome int result; 927afc2ba1dSToomas Soome 928afc2ba1dSToomas Soome /* 929afc2ba1dSToomas Soome * out of kernel memory, gently fall through - this will disable 930afc2ba1dSToomas Soome * compression in zio_compress_data 931afc2ba1dSToomas Soome */ 932afc2ba1dSToomas Soome if (ctx == NULL) 933afc2ba1dSToomas Soome return (0); 934afc2ba1dSToomas Soome 935afc2ba1dSToomas Soome if (isize < LZ4_64KLIMIT) 936afc2ba1dSToomas Soome result = LZ4_compress64kCtx(ctx, source, dest, isize, osize); 937afc2ba1dSToomas Soome else 938afc2ba1dSToomas Soome result = LZ4_compressCtx(ctx, source, dest, isize, osize); 939afc2ba1dSToomas Soome 940*10ae99eeSToomas Soome #if defined(_KERNEL) || defined(_FAKE_KERNEL) 941*10ae99eeSToomas Soome kmem_free(ctx, sizeof (struct refTables)); 942*10ae99eeSToomas Soome #else 943*10ae99eeSToomas Soome free(ctx); 944*10ae99eeSToomas Soome #endif 945afc2ba1dSToomas Soome return (result); 946afc2ba1dSToomas Soome #else 947afc2ba1dSToomas Soome if (isize < (int)LZ4_64KLIMIT) 948afc2ba1dSToomas Soome return (LZ4_compress64kCtx(NULL, source, dest, isize, osize)); 949afc2ba1dSToomas Soome return (LZ4_compressCtx(NULL, source, dest, isize, osize)); 950afc2ba1dSToomas Soome #endif 951afc2ba1dSToomas Soome } 952afc2ba1dSToomas Soome 953afc2ba1dSToomas Soome /* Decompression functions */ 954afc2ba1dSToomas Soome 955afc2ba1dSToomas Soome /* 956afc2ba1dSToomas Soome * Note: The decoding function LZ4_uncompress_unknownOutputSize() is safe 957*10ae99eeSToomas Soome * against "buffer overflow" attack type. It will never write nor 958*10ae99eeSToomas Soome * read outside of the provided output buffers. 959*10ae99eeSToomas Soome * LZ4_uncompress_unknownOutputSize() also insures that 960*10ae99eeSToomas Soome * it will never read outside of the input buffer. A corrupted input 961*10ae99eeSToomas Soome * will produce an error result, a negative int, indicating the position 962*10ae99eeSToomas Soome * of the error within input stream. 963afc2ba1dSToomas Soome */ 964afc2ba1dSToomas Soome 965afc2ba1dSToomas Soome static int 966afc2ba1dSToomas Soome LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize, 967afc2ba1dSToomas Soome int maxOutputSize) 968afc2ba1dSToomas Soome { 969afc2ba1dSToomas Soome /* Local Variables */ 970afc2ba1dSToomas Soome const BYTE *restrict ip = (const BYTE *) source; 971afc2ba1dSToomas Soome const BYTE *const iend = ip + isize; 972afc2ba1dSToomas Soome const BYTE *ref; 973afc2ba1dSToomas Soome 974afc2ba1dSToomas Soome BYTE *op = (BYTE *) dest; 975afc2ba1dSToomas Soome BYTE *const oend = op + maxOutputSize; 976afc2ba1dSToomas Soome BYTE *cpy; 977afc2ba1dSToomas Soome 978afc2ba1dSToomas Soome size_t dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0}; 979afc2ba1dSToomas Soome #if LZ4_ARCH64 980afc2ba1dSToomas Soome size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; 981afc2ba1dSToomas Soome #endif 982afc2ba1dSToomas Soome 983afc2ba1dSToomas Soome /* Main Loop */ 984afc2ba1dSToomas Soome while (ip < iend) { 985afc2ba1dSToomas Soome unsigned token; 986afc2ba1dSToomas Soome size_t length; 987afc2ba1dSToomas Soome 988afc2ba1dSToomas Soome /* get runlength */ 989afc2ba1dSToomas Soome token = *ip++; 990afc2ba1dSToomas Soome if ((length = (token >> ML_BITS)) == RUN_MASK) { 991afc2ba1dSToomas Soome int s = 255; 992afc2ba1dSToomas Soome while ((ip < iend) && (s == 255)) { 993afc2ba1dSToomas Soome s = *ip++; 994afc2ba1dSToomas Soome length += s; 995afc2ba1dSToomas Soome } 996afc2ba1dSToomas Soome } 997afc2ba1dSToomas Soome /* copy literals */ 998afc2ba1dSToomas Soome cpy = op + length; 999afc2ba1dSToomas Soome /* CORNER-CASE: cpy might overflow. */ 1000afc2ba1dSToomas Soome if (cpy < op) 1001afc2ba1dSToomas Soome goto _output_error; /* cpy was overflowed, bail! */ 1002afc2ba1dSToomas Soome if ((cpy > oend - COPYLENGTH) || 1003afc2ba1dSToomas Soome (ip + length > iend - COPYLENGTH)) { 1004afc2ba1dSToomas Soome if (cpy > oend) 1005afc2ba1dSToomas Soome /* Error: writes beyond output buffer */ 1006afc2ba1dSToomas Soome goto _output_error; 1007afc2ba1dSToomas Soome if (ip + length != iend) 1008afc2ba1dSToomas Soome /* 1009afc2ba1dSToomas Soome * Error: LZ4 format requires to consume all 1010afc2ba1dSToomas Soome * input at this stage 1011afc2ba1dSToomas Soome */ 1012afc2ba1dSToomas Soome goto _output_error; 1013afc2ba1dSToomas Soome (void) memcpy(op, ip, length); 1014afc2ba1dSToomas Soome op += length; 1015afc2ba1dSToomas Soome /* Necessarily EOF, due to parsing restrictions */ 1016afc2ba1dSToomas Soome break; 1017afc2ba1dSToomas Soome } 1018afc2ba1dSToomas Soome LZ4_WILDCOPY(ip, op, cpy); 1019afc2ba1dSToomas Soome ip -= (op - cpy); 1020afc2ba1dSToomas Soome op = cpy; 1021afc2ba1dSToomas Soome 1022afc2ba1dSToomas Soome /* get offset */ 1023afc2ba1dSToomas Soome LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip); 1024afc2ba1dSToomas Soome ip += 2; 1025afc2ba1dSToomas Soome if (ref < (BYTE * const) dest) 1026afc2ba1dSToomas Soome /* 1027afc2ba1dSToomas Soome * Error: offset creates reference outside of 1028afc2ba1dSToomas Soome * destination buffer 1029afc2ba1dSToomas Soome */ 1030afc2ba1dSToomas Soome goto _output_error; 1031afc2ba1dSToomas Soome 1032afc2ba1dSToomas Soome /* get matchlength */ 1033afc2ba1dSToomas Soome if ((length = (token & ML_MASK)) == ML_MASK) { 1034afc2ba1dSToomas Soome while (ip < iend) { 1035afc2ba1dSToomas Soome int s = *ip++; 1036afc2ba1dSToomas Soome length += s; 1037afc2ba1dSToomas Soome if (s == 255) 1038afc2ba1dSToomas Soome continue; 1039afc2ba1dSToomas Soome break; 1040afc2ba1dSToomas Soome } 1041afc2ba1dSToomas Soome } 1042afc2ba1dSToomas Soome /* copy repeated sequence */ 1043afc2ba1dSToomas Soome if unlikely(op - ref < STEPSIZE) { 1044afc2ba1dSToomas Soome #if LZ4_ARCH64 1045afc2ba1dSToomas Soome size_t dec64 = dec64table[op-ref]; 1046afc2ba1dSToomas Soome #else 1047afc2ba1dSToomas Soome const int dec64 = 0; 1048afc2ba1dSToomas Soome #endif 1049afc2ba1dSToomas Soome op[0] = ref[0]; 1050afc2ba1dSToomas Soome op[1] = ref[1]; 1051afc2ba1dSToomas Soome op[2] = ref[2]; 1052afc2ba1dSToomas Soome op[3] = ref[3]; 1053afc2ba1dSToomas Soome op += 4; 1054afc2ba1dSToomas Soome ref += 4; 1055afc2ba1dSToomas Soome ref -= dec32table[op-ref]; 1056afc2ba1dSToomas Soome A32(op) = A32(ref); 1057afc2ba1dSToomas Soome op += STEPSIZE - 4; 1058afc2ba1dSToomas Soome ref -= dec64; 1059afc2ba1dSToomas Soome } else { 1060afc2ba1dSToomas Soome LZ4_COPYSTEP(ref, op); 1061afc2ba1dSToomas Soome } 1062afc2ba1dSToomas Soome cpy = op + length - (STEPSIZE - 4); 1063afc2ba1dSToomas Soome if (cpy > oend - COPYLENGTH) { 1064afc2ba1dSToomas Soome if (cpy > oend) 1065afc2ba1dSToomas Soome /* 1066afc2ba1dSToomas Soome * Error: request to write outside of 1067afc2ba1dSToomas Soome * destination buffer 1068afc2ba1dSToomas Soome */ 1069afc2ba1dSToomas Soome goto _output_error; 1070afc2ba1dSToomas Soome LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH)); 1071afc2ba1dSToomas Soome while (op < cpy) 1072afc2ba1dSToomas Soome *op++ = *ref++; 1073afc2ba1dSToomas Soome op = cpy; 1074afc2ba1dSToomas Soome if (op == oend) 1075afc2ba1dSToomas Soome /* 1076afc2ba1dSToomas Soome * Check EOF (should never happen, since 1077afc2ba1dSToomas Soome * last 5 bytes are supposed to be literals) 1078afc2ba1dSToomas Soome */ 1079afc2ba1dSToomas Soome goto _output_error; 1080afc2ba1dSToomas Soome continue; 1081afc2ba1dSToomas Soome } 1082afc2ba1dSToomas Soome LZ4_SECURECOPY(ref, op, cpy); 1083afc2ba1dSToomas Soome op = cpy; /* correction */ 1084afc2ba1dSToomas Soome } 1085afc2ba1dSToomas Soome 1086afc2ba1dSToomas Soome /* end of decoding */ 1087afc2ba1dSToomas Soome return (int)(((char *)op) - dest); 1088afc2ba1dSToomas Soome 1089afc2ba1dSToomas Soome /* write overflow error detected */ 1090afc2ba1dSToomas Soome _output_error: 1091*10ae99eeSToomas Soome return (int)(-(((const char *)ip) - source)); 1092afc2ba1dSToomas Soome } 1093