Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fine tuning performance further (#526)

Browse Source 
* final comit

changing using mappedbytebuffer

changes before using unsafe address

using unsafe

* using graalvm,correct unsafe mem implementation

---------

Co-authored-by: Karthikeyans <karthikeyan.sn@zohocorp.com>
This commit is contained in:
karthikeyan97 2024-01-24 00:51:52 +05:30 committed by GitHub
parent 6c0949969a
commit 292edc629f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 50 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
calculate_average_karthikeyan97.sh
View File

@ -15,15 +15,15 @@
# limitations under the License. # limitations under the License.
# #
JAVA_OPTS="-Xms20480m -Xmx40960m " JAVA_OPTS="-Xms10240m -Xmx40960m "
if [ -f target/CalculateAverage_karthikeyan97_image ]; then if [ -f target/CalculateAverage_karthikeyan97_image ]; then
#echo "Picking up existing native image 'target/CalculateAverage_karthikeyan97_image', delete the file to select JVM mode." 1>&2 #echo "Picking up existing native image 'target/CalculateAverage_karthikeyan97_image', delete the file to select JVM mode." 1>&2
target/CalculateAverage_karthikeyan97_image -Xms20480m -Xmx32768m target/CalculateAverage_karthikeyan97_image -Xms10240m -Xmx40960m
else else
JAVA_OPTS="--enable-preview"
#echo "Chosing to run the app in JVM mode as no native image was found, use prepare_karthikeyan97.sh to generate." 1>&2 #echo "Chosing to run the app in JVM mode as no native image was found, use prepare_karthikeyan97.sh to generate." 1>&2
java -Xms20480m -Xmx32768m --enable-preview --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_karthikeyan97 java -Xms10240m -Xmx40960m --enable-preview --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_karthikeyan97
fi fi

101
src/main/java/dev/morling/onebrc/CalculateAverage_karthikeyan97.java
View File

@ -125,7 +125,7 @@ public class CalculateAverage_karthikeyan97 {
final long mappedAddress = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, raf.length(), Arena.global()).address(); final long mappedAddress = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0, raf.length(), Arena.global()).address();
long length = raf.length(); long length = raf.length();
final long endAddress = mappedAddress + length - 1; final long endAddress = mappedAddress + length - 1;
int cores = length > 1000 ? Runtime.getRuntime().availableProcessors() * 2 : 1; int cores = length > 1000 ? Runtime.getRuntime().availableProcessors() : 1;
long boundary[][] = new long[cores][2]; long boundary[][] = new long[cores][2];
long segments = length / (cores); long segments = length / (cores);
long before = -1; long before = -1;
@ -145,25 +145,21 @@ public class CalculateAverage_karthikeyan97 {
boundary[cores - 1][0] = before + 1; boundary[cores - 1][0] = before + 1;
boundary[cores - 1][1] = length - 1; boundary[cores - 1][1] = length - 1;
Field f = Unsafe.class.getDeclaredField("theUnsafe"); int l3Size = (12 * 1024 * 1024);// unsafe.l3Size();
f.setAccessible(true);
Unsafe unsafe = (Unsafe) f.get(null);
int l3Size = (13 * 1024 * 1024);// unsafe.l3Size();
System.out.println(new TreeMap((Arrays.stream(boundary).parallel().map(i -> { System.out.println(new TreeMap((Arrays.stream(boundary).parallel().map(i -> {
FileInputStream fileInputStream = null;
try { try {
int seglen = (int) (i[1] - i[0] + 1); int seglen = (int) (i[1] - i[0] + 1);
HashMap<modifiedbytearray, MeasurementAggregator> resultmap = new HashMap<>(1000); HashMap<modifiedbytearray, MeasurementAggregator> resultmap = new HashMap<>(4000);
long segstart = mappedAddress + i[0]; long segstart = mappedAddress + i[0];
int bytesRemaining = seglen; int bytesRemaining = seglen;
long num = 0; long num = 0;
int sign = 1;
boolean isNumber = false; boolean isNumber = false;
byte bi; byte bi;
int sign = 1;
modifiedbytearray stationName = null; modifiedbytearray stationName = null;
int hascode = 5381; int hascode = 5381;
// System.out.println("start:" + System.nanoTime() / 1000000);
while (bytesRemaining > 0) { while (bytesRemaining > 0) {
int bytesptr = 0; int bytesptr = 0;
// int bytesread = buffer.remaining() > l3Size ? l3Size : buffer.remaining(); // int bytesread = buffer.remaining() > l3Size ? l3Size : buffer.remaining();
@ -178,64 +174,59 @@ public class CalculateAverage_karthikeyan97 {
while (bytesptr < actualReadSize) { while (bytesptr < actualReadSize) {
bi = readArr[bytesptr++];// UNSAFE.getByte(segstart + bytesReading++); bi = readArr[bytesptr++];// UNSAFE.getByte(segstart + bytesReading++);
if (!isNumber) { if (!isNumber) {
if (bi >= 192) { while (bi != 59) {
hascode = (hascode << 5) + hascode ^ bi;
}
else if (bi == 59) {
isNumber = true;
stationName = new modifiedbytearray(readArr, bbstart, bytesptr - 2, hascode & 0xFFFFFFFF);
bbstart = 0;
hascode = 5381;
if (bytesptr >= readSize) {
break;
}
}
else {
hascode = (hascode << 5) + hascode ^ bi; hascode = (hascode << 5) + hascode ^ bi;
bi = readArr[bytesptr++];
} }
isNumber = true;
stationName = new modifiedbytearray(readArr, bbstart, bytesptr - 2, hascode & 0xFFFFFFFF);
bbstart = 0;
hascode = 5381;
} }
else { else {
switch (bi) { while (bi != 10) {
case 0x2E: if (bi == 0x2D) {
break;
case 0x2D:
sign = -1; sign = -1;
break; }
case 10: else if (bi != 0x2E) {
hascode = 5381;
isNumber = false;
bbstart = bytesptr;
MeasurementAggregator agg = resultmap.get(stationName);
num *= sign;
if (agg == null) {
agg = new MeasurementAggregator();
agg.min = num;
agg.max = num;
agg.sum = (long) (num);
agg.count = 1;
resultmap.put(stationName, agg);
}
else {
if (agg.min >= num) {
agg.min = num;
}
if (agg.max <= num) {
agg.max = num;
}
agg.sum += (long) (num);
agg.count++;
}
num = 0;
sign = 1;
break;
default:
num = num * 10 + (bi - 0x30); num = num * 10 + (bi - 0x30);
}
bi = readArr[bytesptr++];
}
hascode = 5381;
isNumber = false;
bbstart = bytesptr;
num *= sign;
MeasurementAggregator agg = resultmap.get(stationName);
if (agg == null) {
agg = new MeasurementAggregator();
agg.min = num;
agg.max = num;
agg.sum = (long) (num);
agg.count = 1;
resultmap.put(stationName, agg);
}
else {
if (agg.min >= num) {
agg.min = num;
}
if (agg.max <= num) {
agg.max = num;
}
agg.sum += (long) (num);
agg.count++;
}
num = 0;
sign = 1;
if (bytesptr >= readSize) {
break;
} }
} }
} }
bytesRemaining -= bytesptr; bytesRemaining -= bytesptr;
segstart += bytesptr; segstart += bytesptr;
} }
// System.out.println("end:" + System.nanoTime() / 1000000);
/* /*
* while (bytesReading < (i[1] - i[0] + 1) && buffer.position() < buffer.limit()) { * while (bytesReading < (i[1] - i[0] + 1) && buffer.position() < buffer.limit()) {
* buffer.clear(); * buffer.clear();