diff --git a/.github/workflows/build-native.yml b/.github/workflows/build-native.yml index 40839a9f..ccadc181 100644 --- a/.github/workflows/build-native.yml +++ b/.github/workflows/build-native.yml @@ -2,9 +2,30 @@ name: Build Native on: workflow_dispatch: + push: + branches: + - master + - main + paths: + - 'src/main/resources/org/xerial/snappy/VERSION' + - 'Makefile' + - 'Makefile.common' + - '**/*.h' + - '**/*.cpp' + - .github/workflows/build-native.yml + pull_request: + paths: + - 'src/main/resources/org/xerial/snappy/VERSION' + - 'Makefile' + - 'Makefile.common' + - '**/*.h' + - '**/*.cpp' jobs: build: + permissions: + contents: write + pull-requests: write name: Build native libraries runs-on: ubuntu-latest steps: @@ -13,25 +34,11 @@ jobs: run: make clean-native native-all env: OCI_EXE: docker - - name: Upload native libraries - uses: actions/upload-artifact@v3 - with: - name: native-libs - path: src/main/resources/org/xerial/snappy/native/ - push: - name: Push new native libraries to branch - runs-on: ubuntu-latest - needs: [build] - steps: - - uses: actions/checkout@v3 - - name: Download native libraries - uses: actions/download-artifact@v3 - with: - name: native-libs - path: src/main/resources/org/xerial/snappy/native/ - - run: git status - - name: Commit and push - uses: EndBug/add-and-commit@v9 + - name: Create Pull Request + if: ${{ github.event_name != 'pull_request' }} + uses: peter-evans/create-pull-request@v5 with: - message: 'Update native libraries' - default_author: github_actions + title: Update native libraries + commit-message: Update native libraries for ${{ github.sha }} + branch: update-native-libs + labels: library-update diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fbbeafc6..a857ff15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,14 +7,21 @@ on: - '**.java' - '**.sbt' - '.github/workflows/*.yml' + - '**.so' + - '**.dll' + - 'src/main/resources/**' + - 'project/build.properties' push: branches: - master + - main paths: - '**.scala' - '**.java' - '**.sbt' - '.github/workflows/*.yml' + - 'src/main/resources/org/xerial/snappy/**' + - 'project/build.properties' jobs: code_format: diff --git a/.scalafmt.conf b/.scalafmt.conf index c40e9b32..60f85233 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = 3.7.3 +version = 3.7.4 runner.dialect = scala213source3 maxColumn = 180 style = defaultWithAlign diff --git a/Makefile b/Makefile index 2d2ed082..adba9cad 100644 --- a/Makefile +++ b/Makefile @@ -203,6 +203,8 @@ freebsd64: # For ARM native-arm: linux-arm64 linux-android-arm linux-android-aarch64 linux-arm linux-armv6 linux-armv7 + +# TODO: CROSS_PREFIX can be replaced with ${CROSS_ROOT}/bin/${CROSS_TRIPLE}- in Makefile.common linux-arm: jni-header ./docker/dockcross-armv5 -a $(DOCKER_RUN_OPTS) bash -c 'make clean-native native CROSS_PREFIX=/usr/xcc/armv5-unknown-linux-gnueabi/bin//armv5-unknown-linux-gnueabi- OS_NAME=Linux OS_ARCH=arm' diff --git a/Makefile.common b/Makefile.common index 69bed023..8c060e35 100755 --- a/Makefile.common +++ b/Makefile.common @@ -50,6 +50,7 @@ endif # os=Default is meant to be generic unix/linux +# To support new CPU architecture, add a new target name here known_os_archs := Linux-x86 Linux-x86_64 Linux-arm Linux-armv6 Linux-armv7 Linux-android-arm Linux-android-aarch64 Linux-aarch64 Linux-ppc Linux-ppc64 Linux-ppc64le Linux-s390 Linux-s390x Mac-x86 Mac-x86_64 Mac-aarch64 FreeBSD-x86_64 Windows-x86 Windows-x86_64 SunOS-x86 SunOS-sparc SunOS-x86_64 AIX-ppc AIX-ppc64 Linux-riscv Linux-riscv64 os_arch := $(OS_NAME)-$(OS_ARCH) IBM_JDK_7 := $(findstring IBM, $(shell $(JAVA) -version 2>&1 | grep IBM | grep "JRE 1.7")) diff --git a/README.md b/README.md index b918aff0..e33a56cd 100755 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ snappy-java uses sbt (simple build tool for Scala) as a build tool. Here is a si $ ./sbt # enter sbt console > ~test # run tests upon source code change - > ~test-only * # run tests that matches a given name pattern + > ~testOnly # run tests that matches a given name pattern > publishM2 # publish jar to $HOME/.m2/repository > package # create jar file > findbugs # Produce findbugs report in target/findbugs diff --git a/build.sbt b/build.sbt index 8bd38310..13efd7a2 100644 --- a/build.sbt +++ b/build.sbt @@ -75,7 +75,7 @@ libraryDependencies ++= Seq( "junit" % "junit" % "4.13.2" % "test", "org.codehaus.plexus" % "plexus-classworlds" % "2.7.0" % "test", "org.xerial.java" % "xerial-core" % "2.1" % "test", - "org.wvlet.airframe" %% "airframe-log" % "23.5.5" % "test", + "org.wvlet.airframe" %% "airframe-log" % "23.5.6" % "test", "org.osgi" % "org.osgi.core" % "6.0.0" % "provided", "com.github.sbt" % "junit-interface" % "0.13.3" % "test", "org.apache.hadoop" % "hadoop-common" % "2.10.2" % "test" exclude ("org.xerial.snappy", "snappy-java") diff --git a/project/build.properties b/project/build.properties index 5041518d..01892960 100755 --- a/project/build.properties +++ b/project/build.properties @@ -1,2 +1,2 @@ -sbt.version=1.8.3 +sbt.version=1.9.0 diff --git a/src/main/java/org/xerial/snappy/BitShuffle.java b/src/main/java/org/xerial/snappy/BitShuffle.java index fa623478..b2a4a6fc 100644 --- a/src/main/java/org/xerial/snappy/BitShuffle.java +++ b/src/main/java/org/xerial/snappy/BitShuffle.java @@ -91,6 +91,9 @@ public static int shuffle(ByteBuffer input, BitShuffleType type, ByteBuffer shuf * @throws IOException */ public static byte[] shuffle(short[] input) throws IOException { + if (input.length * 2 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 2]; int numProcessed = impl.shuffle(input, 0, 2, input.length * 2, output, 0); assert(numProcessed == input.length * 2); @@ -105,6 +108,9 @@ public static byte[] shuffle(short[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(int[] input) throws IOException { + if (input.length * 4 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 4]; int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0); assert(numProcessed == input.length * 4); @@ -119,6 +125,9 @@ public static byte[] shuffle(int[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(long[] input) throws IOException { + if (input.length * 8 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 8]; int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0); assert(numProcessed == input.length * 8); @@ -133,6 +142,9 @@ public static byte[] shuffle(long[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(float[] input) throws IOException { + if (input.length * 4 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 4]; int numProcessed = impl.shuffle(input, 0, 4, input.length * 4, output, 0); assert(numProcessed == input.length * 4); @@ -147,6 +159,9 @@ public static byte[] shuffle(float[] input) throws IOException { * @throws IOException */ public static byte[] shuffle(double[] input) throws IOException { + if (input.length * 8 < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } byte[] output = new byte[input.length * 8]; int numProcessed = impl.shuffle(input, 0, 8, input.length * 8, output, 0); assert(numProcessed == input.length * 8); diff --git a/src/main/java/org/xerial/snappy/Snappy.java b/src/main/java/org/xerial/snappy/Snappy.java index 5263e94b..14d17964 100755 --- a/src/main/java/org/xerial/snappy/Snappy.java +++ b/src/main/java/org/xerial/snappy/Snappy.java @@ -169,7 +169,11 @@ public static int compress(ByteBuffer uncompressed, ByteBuffer compressed) public static byte[] compress(char[] input) throws IOException { - return rawCompress(input, input.length * 2); // char uses 2 bytes + int byteSize = input.length * 2; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // char uses 2 bytes } /** @@ -181,7 +185,11 @@ public static byte[] compress(char[] input) public static byte[] compress(double[] input) throws IOException { - return rawCompress(input, input.length * 8); // double uses 8 bytes + int byteSize = input.length * 8; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // double uses 8 bytes } /** @@ -193,7 +201,11 @@ public static byte[] compress(double[] input) public static byte[] compress(float[] input) throws IOException { - return rawCompress(input, input.length * 4); // float uses 4 bytes + int byteSize = input.length * 4; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // float uses 4 bytes } /** @@ -205,7 +217,11 @@ public static byte[] compress(float[] input) public static byte[] compress(int[] input) throws IOException { - return rawCompress(input, input.length * 4); // int uses 4 bytes + int byteSize = input.length * 4; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // int uses 4 bytes } /** @@ -217,7 +233,11 @@ public static byte[] compress(int[] input) public static byte[] compress(long[] input) throws IOException { - return rawCompress(input, input.length * 8); // long uses 8 bytes + int byteSize = input.length * 8; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // long uses 8 bytes } /** @@ -229,7 +249,11 @@ public static byte[] compress(long[] input) public static byte[] compress(short[] input) throws IOException { - return rawCompress(input, input.length * 2); // short uses 2 bytes + int byteSize = input.length * 2; + if (byteSize < input.length) { + throw new SnappyError(SnappyErrorCode.TOO_LARGE_INPUT, "input array size is too large: " + input.length); + } + return rawCompress(input, byteSize); // short uses 2 bytes } /** diff --git a/src/main/java/org/xerial/snappy/SnappyErrorCode.java b/src/main/java/org/xerial/snappy/SnappyErrorCode.java index 24d5b18a..661ffd84 100755 --- a/src/main/java/org/xerial/snappy/SnappyErrorCode.java +++ b/src/main/java/org/xerial/snappy/SnappyErrorCode.java @@ -42,7 +42,8 @@ public enum SnappyErrorCode EMPTY_INPUT(6), INCOMPATIBLE_VERSION(7), INVALID_CHUNK_SIZE(8), - UNSUPPORTED_PLATFORM(9); + UNSUPPORTED_PLATFORM(9), + TOO_LARGE_INPUT(10); public final int id; diff --git a/src/main/java/org/xerial/snappy/SnappyInputStream.java b/src/main/java/org/xerial/snappy/SnappyInputStream.java index 24f59767..9835cf90 100755 --- a/src/main/java/org/xerial/snappy/SnappyInputStream.java +++ b/src/main/java/org/xerial/snappy/SnappyInputStream.java @@ -417,9 +417,20 @@ protected boolean hasNextChunk() } } + // chunkSize is negative + if (chunkSize < 0) { + throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, "chunkSize is too big or negative : " + chunkSize); + } + // extend the compressed data buffer size if (compressed == null || chunkSize > compressed.length) { - compressed = new byte[chunkSize]; + // chunkSize exceeds limit + try { + compressed = new byte[chunkSize]; + } + catch (java.lang.OutOfMemoryError e) { + throw new SnappyError(SnappyErrorCode.INVALID_CHUNK_SIZE, e.getMessage()); + } } readBytes = 0; while (readBytes < chunkSize) { diff --git a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll index 840d7d81..2076a640 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86/snappyjava.dll differ diff --git a/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll index cb6c7703..6269de2d 100755 Binary files a/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll and b/src/main/resources/org/xerial/snappy/native/Windows/x86_64/snappyjava.dll differ diff --git a/src/test/java/org/xerial/snappy/SnappyTest.java b/src/test/java/org/xerial/snappy/SnappyTest.java index 18b39e92..30edf66c 100755 --- a/src/test/java/org/xerial/snappy/SnappyTest.java +++ b/src/test/java/org/xerial/snappy/SnappyTest.java @@ -19,16 +19,18 @@ // SnappyTest.java // Since: 2011/03/30 // -// $URL$ +// $URL$ // $Author$ //-------------------------------------- package org.xerial.snappy; import static org.junit.Assert.*; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; +import org.junit.Assume; import org.junit.Assert; import org.junit.Test; import org.xerial.util.log.Logger; @@ -329,4 +331,187 @@ public void isValidCompressedData() _logger.debug(e); } } + + + /* + Tests happy cases for SnappyInputStream.read method + - {0} + */ + @Test + public void isValidChunkLengthForSnappyInputStreamIn() + throws Exception { + byte[] data = {0}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + in.read(out); + } + + /* + Tests sad cases for SnappyInputStream.read method + - Expects a java.lang.NegativeArraySizeException catched into a SnappyError + - {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff} + */ + @Test(expected = SnappyError.class) + public void isInvalidChunkLengthForSnappyInputStreamInNegative() + throws Exception { + byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + in.read(out); + } + + /* + Tests sad cases for SnappyInputStream.read method + - Expects a java.lang.OutOfMemoryError + - {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0,(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff} + */ + @Test(expected = SnappyError.class) + public void isInvalidChunkLengthForSnappyInputStreamOutOfMemory() + throws Exception { + byte[] data = {-126, 'S', 'N', 'A', 'P', 'P', 'Y', 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff}; + SnappyInputStream in = new SnappyInputStream(new ByteArrayInputStream(data)); + byte[] out = new byte[50]; + try { + in.read(out); + } catch (Exception ignored) { + // Exception here will be catched + // But OutOfMemoryError will not be caught, and will still be thrown + } + } + + /* + Tests happy cases for BitShuffle.shuffle method + - double: 0, 10 + - float: 0, 10 + - int: 0, 10 + - long: 0, 10 + - short: 0, 10 + */ + @Test + public void isValidArrayInputLength() + throws Exception { + byte[] a = Snappy.compress(new char[0]); + byte[] b = Snappy.compress(new double[0]); + byte[] c = Snappy.compress(new float[0]); + byte[] d = Snappy.compress(new int[0]); + byte[] e = Snappy.compress(new long[0]); + byte[] f = Snappy.compress(new short[0]); + byte[] g = Snappy.compress(new char[10]); + byte[] h = Snappy.compress(new double[10]); + byte[] i = Snappy.compress(new float[10]); + byte[] j = Snappy.compress(new int[10]); + byte[] k = Snappy.compress(new long[10]); + byte[] l = Snappy.compress(new short[10]); + } + + /* + Tests sad cases for Snappy.compress + - Allocate a buffer whose byte size will be a bit larger than Integer.MAX_VALUE + - char + - double + - float + - int + - long + - short + */ + @Test(expected = SnappyError.class) + public void isTooLargeDoubleArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new double[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeCharArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new char[Integer.MAX_VALUE / 2 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeFloatArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeIntArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new int[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeLongArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new long[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeShortArrayInputLength() throws Exception { + assumingCIIsFalse(); + Snappy.compress(new short[Integer.MAX_VALUE / 2 + 1]); + } + + /* + Tests happy cases for Snappy.compress + - char: 0, 10 + */ + @Test + public void isValidArrayInputLengthForBitShuffleShuffle() + throws Exception + { + byte[] b = BitShuffle.shuffle(new double[0]); + byte[] c = BitShuffle.shuffle(new float[0]); + byte[] d = BitShuffle.shuffle(new int[0]); + byte[] e = BitShuffle.shuffle(new long[0]); + byte[] f = BitShuffle.shuffle(new short[0]); + byte[] n = BitShuffle.shuffle(new double[10]); + byte[] o = BitShuffle.shuffle(new float[10]); + byte[] p = BitShuffle.shuffle(new int[10]); + byte[] q = BitShuffle.shuffle(new long[10]); + byte[] r = BitShuffle.shuffle(new short[10]); + } + + /* + Tests sad cases for BitShuffle.shuffle method + - Allocate a buffer whose byte size will be a bit larger than Integer.MAX_VALUE + - double: 8 + - float: 4 + - int: 4 + - long: 8 + - short: 2 + */ + @Test(expected = SnappyError.class) + public void isTooLargeDoubleArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new double[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeFloatArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeIntArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new float[Integer.MAX_VALUE / 4 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeLongArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new long[Integer.MAX_VALUE / 8 + 1]); + } + + @Test(expected = SnappyError.class) + public void isTooLargeShortArrayInputLengthForBitShuffleShuffle() throws Exception { + assumingCIIsFalse(); + BitShuffle.shuffle(new short[Integer.MAX_VALUE / 2 + 1]); + } + + private void assumingCIIsFalse() { + if (System.getenv("CI") == null) + return; + Assume.assumeFalse("Skipped on CI", System.getenv("CI").equals("true")); + } }