Skip to content

Changes to allow cross-language binaries to be the same as C++. #771

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leerho merged 2 commits into from
Sep 26, 2026
Merged

Changes to allow cross-language binaries to be the same as C++. #771

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a9954b9
Changes to allow cross-language binaries to be the same as C++.
leerho Sep 24, 2026
9951cf8
Improved Cross-lang tests
leerho Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,15 @@ under the License.
<testng.check_cpp_files>check_cpp_files</testng.check_cpp_files>
<testng.check_go_files>check_go_files</testng.check_go_files>
<testng.check_rust_files>check_rust_files</testng.check_rust_files>
<testng.check_cpp_historical_files>check_cpp_historical_files</testng.check_cpp_historical_files>
<testng.all_groups>${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files},${testng.check_go_files},${testng.check_rust_files},${testng.check_cpp_historical_files}</testng.all_groups>
<!-- special groups -->
<testng.check_CL_binaries>check_CL_binary_files</testng.check_CL_binaries>
<testng.check_cpp_historicals>check_cpp_historical_files</testng.check_cpp_historicals>
<!-- Language Groups -->
<testng.check_java_cpp>${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files}</testng.check_java_cpp>
<testng.check_go_rust>${testng.check_go_files},${testng.check_rust_files}</testng.check_go_rust>
<testng.check_specials>${testng.check_CL_binaries}${testng.check_cpp_historicals}</testng.check_specials>
<!-- All Languages & types -->
<testng.all_groups>${testng.check_java_cpp}${testng.check_go_rust}${testng.check_specials}</testng.all_groups>

<test.data.root>${project.basedir}</test.data.root>
<print>false</print> <!-- diagnostic printing of file names read/written/not-found in UtilityIO; override with -Dprint=true -->
Expand Down Expand Up @@ -855,6 +862,22 @@ under the License.
</build>
</profile>

<profile>
<id>check_CL_binary_files</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<groups>${testng.check_CL_binary_files}</groups>
<excludedGroups combine.self="override"></excludedGroups>
</configuration>
</plugin>
</plugins>
</build>
</profile>

<profile>
<id>check_cpp_historical_files</id>
<build>
Expand Down
21 changes: 11 additions & 10 deletions src/main/java/org/apache/datasketches/theta/EmptyCompactSketch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,29 +27,30 @@
import org.apache.datasketches.common.SketchesArgumentException;

/**
* Singleton empty CompactThetaSketch.
* Singleton Empty CompactThetaSketch.
*
* @author Lee Rhodes
*/
final class EmptyCompactSketch extends CompactThetaSketch {

//For backward compatibility, a candidate long must have Flags= compact, read-only,
// COMPACT-Family=3, SerVer=3, PreLongs=1, and be exactly 8 bytes long. The seedHash is ignored.
// NOTE: The empty and ordered flags may or may not be set
private static final long EMPTY_SKETCH_MASK = 0X00_00_EB_00_00_FF_FF_FFL;
private static final long EMPTY_SKETCH_TEST = 0X00_00_0A_00_00_03_03_01L;
//When returning a byte array the empty and ordered bits are also set
// For backward compatibility, a candidate EmptyCompactSketch
// must have Flags: compact, empty, read-only;
// and bytes: COMPACT-Family=3, SerVer=3, PreLongs=1; and be exactly 8 bytes long.
// NOTE: The seedHash bytes are ignored
private static final long EMPTY_SKETCH_MASK = 0X00_00_FF_FF_FF_FF_FF_FFL;
private static final long EMPTY_SKETCH_TEST = 0X00_00_1E_00_00_03_03_01L;
//When returning a byte array the following flags are set: ordered, compact, empty, read-only
static final byte[] EMPTY_COMPACT_SKETCH_ARR = { 1, 3, 3, 0, 0, 0x1E, 0, 0 };
private static final EmptyCompactSketch EMPTY_COMPACT_SKETCH = new EmptyCompactSketch();

private EmptyCompactSketch() {}

static synchronized EmptyCompactSketch getInstance() {
static EmptyCompactSketch getInstance() {
return EMPTY_COMPACT_SKETCH;
}

//This should be a heapify
static synchronized EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) {
//This is effectively a heapify
static EmptyCompactSketch getHeapInstance(final MemorySegment srcSeg) {
final long pre0 = srcSeg.get(JAVA_LONG_UNALIGNED, 0);
if (testCandidatePre0(pre0)) {
return EMPTY_COMPACT_SKETCH;
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/org/apache/datasketches/theta/PreambleUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,12 +204,12 @@ private PreambleUtil() {}
static final int UNION_THETA_LONG = 24; //8-byte aligned, only used by ThetaUnion

// flag byte bit masks
static final int RESERVED_FLAG_MASK = 1; //Bit 0: Reserved, no longer used. Was BigEndian
static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: Reserved, Set but not read.
static final int EMPTY_FLAG_MASK = 4; //Bit 2:
static final int COMPACT_FLAG_MASK = 8; //Bit 3:
static final int ORDERED_FLAG_MASK = 16;//Bit 4:
static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5:
static final int RESERVED_FLAG_MASK = 1; //Bit 0: 0x1: Reserved, no longer used. Was BigEndian
static final int READ_ONLY_FLAG_MASK = 2; //Bit 1: 0x2:
static final int EMPTY_FLAG_MASK = 4; //Bit 2: 0x4:
static final int COMPACT_FLAG_MASK = 8; //Bit 3: 0x8:
static final int ORDERED_FLAG_MASK = 16;//Bit 4: 0x10:
static final int SINGLEITEM_FLAG_MASK = 32;//Bit 5: 0x20:
//The last 2 bits (Bit 6,7) of the flags byte are reserved and assumed to be zero.

//Other constants
Expand Down
3 changes: 2 additions & 1 deletion src/test/java/org/apache/datasketches/common/UtilityIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
* Utilities common to testing
*/
public final class UtilityIO {
private static final String LS = System.getProperty("line.separator");
public static final String LS = System.getProperty("line.separator");
private static final Class<?> clazz = UtilityIO.class;
private static final ClassLoader CL = clazz.getClassLoader();
private static final String TEST_DATA_ROOT_PROP = "test.data.root";
Expand All @@ -44,6 +44,7 @@ public final class UtilityIO {
public static final String CHECK_CPP_FILES = "check_cpp_files";
public static final String CHECK_GO_FILES = "check_go_files";
public static final String CHECK_RUST_FILES = "check_rust_files";
public static final String CHECK_CL_BINARY_FILES = "check_CL_binary_files";
public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files";

/**
Expand Down
43 changes: 43 additions & 0 deletions src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

package org.apache.datasketches.hll;

import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES;
import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES;
import static org.apache.datasketches.common.UtilityIO.LS;
import static org.apache.datasketches.common.UtilityIO.getFileBytes;
import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath;
import static org.apache.datasketches.hll.TgtHllType.HLL_4;
Expand All @@ -34,6 +36,7 @@
import java.io.IOException;
import java.lang.foreign.MemorySegment;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.UtilityIO.GroupLanguage;
import org.testng.annotations.Test;

Expand Down Expand Up @@ -80,6 +83,42 @@
deserializeHll(GroupLanguage.GO);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries1() {
deserializeHll_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries2() {
deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries3() {
deserializeHll_crossBinary(GroupLanguage.GO, GroupLanguage.CPP);
}

private static void deserializeHll_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) {
final String[] sArr = {"hll4", "hll6", "hll8"};
final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
String id = null;
for (final String s: sArr) {
for (final int n: nArr) {
final String fileName1 = s + "_n" + n + lang1.sfx + ".sk";
final byte[] bytes1 = getFileBytes(lang1.pth, fileName1);
final String fileName2 = s + "_n" + n + lang2.sfx + ".sk";
final byte[] bytes2 = getFileBytes(lang2.pth, fileName2);
if (bytes1.length == 0 || bytes2.length == 0) { continue;}
id = "CLB: " + fileName1 + " vs " + fileName2 + ", HllType: " + s + ", n: " + n;
try { assertEquals(bytes1, bytes2); }
catch (final AssertionError e) {
throw new SketchesArgumentException(id + LS + e);
}
println("PASSED: " + id);
}
}
}

private static void deserializeHll(final GroupLanguage lang) {
final String[] sArr = {"hll4", "hll6", "hll8"};
final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
Expand All @@ -97,4 +136,8 @@
}
}

static void println(Object o) {
Comment thread
leerho marked this conversation as resolved.
Dismissed
//System.out.println(o.toString());
}

}
40 changes: 40 additions & 0 deletions src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@

package org.apache.datasketches.theta;

import static org.apache.datasketches.common.UtilityIO.CHECK_CL_BINARY_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES;
import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES;
import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES;
import static org.apache.datasketches.common.UtilityIO.LS;
import static org.apache.datasketches.common.UtilityIO.getFileBytes;
import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath;
import static org.testng.Assert.assertEquals;
Expand All @@ -32,6 +34,7 @@
import java.io.IOException;
import java.lang.foreign.MemorySegment;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.UtilityIO.GroupLanguage;
import org.testng.annotations.Test;

Expand Down Expand Up @@ -104,6 +107,39 @@
deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries1() {
deserializeSketchesFromFile_crossBinary(GroupLanguage.CPP, GroupLanguage.JAVA);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries2() {
deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.JAVA);
}

@Test(groups = {CHECK_CL_BINARY_FILES})
public void checkBinaries3() {
deserializeSketchesFromFile_crossBinary(GroupLanguage.GO, GroupLanguage.CPP);
}

private static void deserializeSketchesFromFile_crossBinary(final GroupLanguage lang1, final GroupLanguage lang2) {
final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
String id = null;
for (final int n: nArr) {
final String fileName1 = "theta_n" + n + lang1.sfx + ".sk";
final byte[] bytes1 = getFileBytes(lang1.pth, fileName1);
final String fileName2 = "theta_n" + n + lang2.sfx + ".sk";
final byte[] bytes2 = getFileBytes(lang2.pth, fileName2);
if (bytes1.length == 0 || bytes2.length == 0) { continue; }
id = "CLB: " + fileName1 + " vs " + fileName2 + ", n: " + n;
try { assertEquals(bytes1, bytes2); }
catch (final AssertionError e) {
throw new SketchesArgumentException(id + LS + e);
}
println("PASSED: " + id);
}
}

private static void deserializeSketchesUsingSegment(final GroupLanguage lang) {
final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
for (final int n: nArr) {
Expand Down Expand Up @@ -208,4 +244,8 @@
assertEquals(sketch.getRetainedEntries(), 0);
}

static void println(Object o) {
Comment thread
leerho marked this conversation as resolved.
Dismissed
//System.out.println(o.toString());
}

}
Loading