diff --git a/pom.xml b/pom.xml index 2792edacf..bdd818a48 100644 --- a/pom.xml +++ b/pom.xml @@ -140,8 +140,15 @@ under the License. check_cpp_files check_go_files check_rust_files - check_cpp_historical_files - ${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files},${testng.check_go_files},${testng.check_rust_files},${testng.check_cpp_historical_files} + + check_CL_binary_files + check_cpp_historical_files + + ${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files} + ${testng.check_go_files},${testng.check_rust_files} + ${testng.check_CL_binaries}${testng.check_cpp_historicals} + + ${testng.check_java_cpp}${testng.check_go_rust}${testng.check_specials} ${project.basedir} false @@ -855,6 +862,22 @@ under the License. + + check_CL_binary_files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.check_CL_binary_files} + + + + + + + check_cpp_historical_files diff --git a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java index a5ffed2f7..1ebc47afe 100644 --- a/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java +++ b/src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java @@ -27,29 +27,30 @@ import org.apache.datasketches.common.SketchesArgumentException; /** - * Singleton empty CompactThetaSketch. + * Singleton Empty CompactThetaSketch. * * @author Lee Rhodes */ final class EmptyCompactSketch extends CompactThetaSketch { - //For backward compatibility, a candidate long must have Flags= compact, read-only, - // COMPACT-Family=3, SerVer=3, PreLongs=1, and be exactly 8 bytes long. The seedHash is ignored. - // NOTE: The empty and ordered flags may or may not be set - private static final long EMPTY_SKETCH_MASK = 0X00_00_EB_00_00_FF_FF_FFL; - private static final long EMPTY_SKETCH_TEST = 0X00_00_0A_00_00_03_03_01L; - //When returning a byte array the empty and ordered bits are also set + // For backward compatibility, a candidate EmptyCompactSketch + // must have Flags: compact, empty, read-only; + // and bytes: COMPACT-Family=3, SerVer=3, PreLongs=1; and be exactly 8 bytes long. + // NOTE: The seedHash bytes are ignored + private static final long EMPTY_SKETCH_MASK = 0X00_00_FF_FF_FF_FF_FF_FFL; + private static final long EMPTY_SKETCH_TEST = 0X00_00_1E_00_00_03_03_01L; + //When returning a byte array the following flags are set: ordered, compact, empty, read-only static final byte[] EMPTY_COMPACT_SKETCH_ARR = { 1, 3, 3, 0, 0, 0x1E, 0, 0 }; private static final EmptyCompactSketch EMPTY_COMPACT_SKETCH = new EmptyCompactSketch(); private EmptyCompactSketch() {} - static synchronized EmptyCompactSketch getInstance() { + static EmptyCompactSketch getInstance() { return EMPTY_COMPACT_SKETCH; } - //This should be a heapify - static synchronized EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { + //This is effectively a heapify + static EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) { final long pre0 = srcSeg.get(JAVA_LONG_UNALIGNED, 0); if (testCandidatePre0(pre0)) { return EMPTY_COMPACT_SKETCH; diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java index fcb7f24a4..f8239511f 100644 --- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java @@ -204,12 +204,12 @@ private PreambleUtil() {} static final int UNION_THETA_LONG = 24; //8-byte aligned, only used by ThetaUnion // flag byte bit masks - static final int RESERVED_FLAG_MASK = 1; //Bit 0: Reserved, no longer used. Was BigEndian - static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: Reserved, Set but not read. - static final int EMPTY_FLAG_MASK = 4; //Bit 2: - static final int COMPACT_FLAG_MASK = 8; //Bit 3: - static final int ORDERED_FLAG_MASK = 16;//Bit 4: - static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: + static final int RESERVED_FLAG_MASK = 1; //Bit 0: 0x1: Reserved, no longer used. Was BigEndian + static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: 0x2: + static final int EMPTY_FLAG_MASK = 4; //Bit 2: 0x4: + static final int COMPACT_FLAG_MASK = 8; //Bit 3: 0x8: + static final int ORDERED_FLAG_MASK = 16;//Bit 4: 0x10: + static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: 0x20: //The last 2 bits (Bit 6,7) of the flags byte are reserved and assumed to be zero. //Other constants diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java index 923dc20c2..f74f58824 100644 --- a/src/test/java/org/apache/datasketches/common/UtilityIO.java +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -31,7 +31,7 @@ * Utilities common to testing */ public final class UtilityIO { - private static final String LS = System.getProperty("line.separator"); + public static final String LS = System.getProperty("line.separator"); private static final Class clazz = UtilityIO.class; private static final ClassLoader CL = clazz.getClassLoader(); private static final String TEST_DATA_ROOT_PROP = "test.data.root"; @@ -44,6 +44,7 @@ public final class UtilityIO { public static final String CHECK_CPP_FILES = "check_cpp_files"; public static final String CHECK_GO_FILES = "check_go_files"; public static final String CHECK_RUST_FILES = "check_rust_files"; + public static final String CHECK_CL_BINARY_FILES = "check_CL_binary_files"; public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; /** diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 8595391a4..f63a50156 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -19,10 +19,12 @@ package org.apache.datasketches.hll; +import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; @@ -34,6 +36,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -80,6 +83,42 @@ public void checkGo() { deserializeHll(GroupLanguage.GO); } + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries1() { + deserializeHll_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries2() { + deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries3() { + deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.CPP); + } + + private static void deserializeHll_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final String[] sArr = {"hll4", "hll6", "hll8"}; + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + String id = null; + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName1 = s + "_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = s + "_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue;} + id = "CLB: " + fileName1 + " vs " + fileName2 + ", HllType: " + s + ", n: " + n; + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + throw new SketchesArgumentException(id + LS + e); + } + println("PASSED: " + id); + } + } + } + private static void deserializeHll(final GroupLanguage lang) { final String[] sArr = {"hll4", "hll6", "hll8"}; final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; @@ -97,4 +136,8 @@ private static void deserializeHll(final GroupLanguage lang) { } } + static void println(Object o) { + //System.out.println(o.toString()); + } + } diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index 5aba332ff..8617cd6b9 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -19,10 +19,12 @@ package org.apache.datasketches.theta; +import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.LS; import static org.apache.datasketches.common.UtilityIO.getFileBytes; import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; @@ -32,6 +34,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; @@ -104,6 +107,39 @@ public void checkGo() { deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); } + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries1() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries2() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CL_BINARY_FILES}) + public void checkBinaries3() { + deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.CPP); + } + + private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) { + final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + String id = null; + for (final int n: nArr) { + final String fileName1 = "theta_n" + n + lang1.sfx + ".sk"; + final byte[] bytes1 = getFileBytes(lang1.pth, fileName1); + final String fileName2 = "theta_n" + n + lang2.sfx + ".sk"; + final byte[] bytes2 = getFileBytes(lang2.pth, fileName2); + if (bytes1.length == 0 || bytes2.length == 0) { continue; } + id = "CLB: " + fileName1 + " vs " + fileName2 + ", n: " + n; + try { assertEquals(bytes1, bytes2); } + catch (final AssertionError e) { + throw new SketchesArgumentException(id + LS + e); + } + println("PASSED: " + id); + } + } + private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { @@ -208,4 +244,8 @@ private static void deserializeNonEmptyNoEntriesFromFile(final GroupLanguage lan assertEquals(sketch.getRetainedEntries(), 0); } + static void println(Object o) { + //System.out.println(o.toString()); + } + }