Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Small improvements (#379)

Browse Source
This commit is contained in:
Jaroslav Bachorik 2024-01-14 14:12:19 +01:00 committed by GitHub
parent f11fad2a68
commit 865188ccee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
calculate_average_jbachorik.sh
View File

@ -15,5 +15,5 @@
# limitations under the License. # limitations under the License.
# #
JAVA_OPTS="-XX:+UseInlineCaches -XX:CICompilerCount=4 -Xmx512m -XX:+UseInlineCaches -XX:MaxInlineSize=512 -XX:FreqInlineSize=800 -XX:InlineSmallCode=190 -XX:CompileThreshold=2" JAVA_OPTS="-Xmx512m -Xms512m"
java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_jbachorik $@ java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_jbachorik $@

2
prepare_jbachorik.sh
View File

@ -16,4 +16,4 @@
# #
source "$HOME/.sdkman/bin/sdkman-init.sh" source "$HOME/.sdkman/bin/sdkman-init.sh"
sdk use java 21.0.1-tem 1>&2 sdk use java 21.0.1-graal 1>&2

116
src/main/java/dev/morling/onebrc/CalculateAverage_jbachorik.java
View File

@ -50,11 +50,11 @@ public class CalculateAverage_jbachorik {
// bb.get(offset, bytes); // bb.get(offset, bytes);
// String str = new String(bytes); // String str = new String(bytes);
if (this.len != len || this.v0 != v0 || this.v1 != v1) { if (((this.len ^ len) | (this.v0 ^ v0) | (this.v1 ^ v1)) != 0) {
return false; return false;
} }
for (int i = 0; i < (len / 8); i++) { for (int i = 0; i < len - 8; i += 8) {
if (bb.getLong(this.offset + i * 8) != bb.getLong(offset + i * 8)) { if (bb.getLong(this.offset + i) != bb.getLong(offset + i)) {
return false; return false;
} }
} }
@ -125,32 +125,26 @@ public class CalculateAverage_jbachorik {
} }
} }
private static final int BUCKETS = 1264532; private static final int BUCKETS = 65536;
private static final int BUCKET_SIZE = 4; private static final int BUCKET_SIZE = 16;
private final StatsHolder[][] map = new StatsHolder[BUCKETS][BUCKET_SIZE]; private final StatsHolder[][] map = new StatsHolder[BUCKETS][BUCKET_SIZE];
public Stats getOrInsert(ByteBuffer buffer, int offset, int len, int hash, long v0, long v1) { public Stats getOrInsert(ByteBuffer buffer, int offset, int len, int idx, long v0, long v1) {
int idx = Math.abs(hash % BUCKETS);
StatsHolder[] bucket = map[idx]; StatsHolder[] bucket = map[idx];
if (bucket[0] == null) {
Stats stats = new Stats();
bucket[0] = new StatsHolder(new Key(buffer, offset, len, v0, v1, hash), stats);
return stats;
}
int bucketOffset = 0; int bucketOffset = 0;
while (bucketOffset < BUCKET_SIZE && bucket[bucketOffset] != null && !bucket[bucketOffset].key.equals(offset, len, v0, v1)) { do {
bucketOffset++; StatsHolder statsHolder = bucket[bucketOffset];
} if (statsHolder == null) {
assert (bucketOffset <= BUCKET_SIZE);
if (bucket[bucketOffset] != null) {
return bucket[bucketOffset].stats;
}
else {
Stats stats = new Stats(); Stats stats = new Stats();
bucket[bucketOffset] = new StatsHolder(new Key(buffer, offset, len, v0, v1, hash), stats); bucket[bucketOffset] = new StatsHolder(new Key(buffer, offset, len, v0, v1, idx), stats);
return stats; return stats;
} }
if (statsHolder.key.equals(offset, len, v0, v1)) {
return statsHolder.stats;
}
bucketOffset++;
} while (bucketOffset < BUCKET_SIZE - 1);
throw new Error("Bucket overflow");
} }
public void forEach(BiConsumer<Key, Stats> consumer) { public void forEach(BiConsumer<Key, Stats> consumer) {
@ -168,19 +162,19 @@ public class CalculateAverage_jbachorik {
private static final long semiPattern = compilePattern((byte) ';'); private static final long semiPattern = compilePattern((byte) ';');
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
int workers = Runtime.getRuntime().availableProcessors() - 1; int workers = Runtime.getRuntime().availableProcessors();
if (args.length == 1) { if (args.length == 1) {
workers = Integer.parseInt(args[0]); workers = Integer.parseInt(args[0]);
} }
Map<String, Stats> map = new TreeMap<>(); Map<String, Stats> map = new TreeMap<>();
File f = new File("measurements.txt"); File f = new File("measurements.txt");
ExecutorService workerPool = Executors.newFixedThreadPool(workers);
ExecutorService mergerPool = Executors.newSingleThreadExecutor();
try (FileInputStream fis = new FileInputStream(f)) { try (FileInputStream fis = new FileInputStream(f)) {
FileChannel fc = fis.getChannel(); FileChannel fc = fis.getChannel();
int granularity = 32 * 1024 * 1024; int granularity = 32 * 1024 * 1024;
int targetWorkers = Math.min(Math.max(1, (int) (fc.size() / granularity)), workers); int targetWorkers = Math.min(Math.max(1, (int) (fc.size() / granularity)), workers);
long chunkSize = fc.size() / targetWorkers; long chunkSize = fc.size() / targetWorkers;
ExecutorService workerPool = Executors.newFixedThreadPool(workers);
// System.out.println("Chunk size: " + chunkSize + ", workers: " + targetWorkers); // System.out.println("Chunk size: " + chunkSize + ", workers: " + targetWorkers);
for (ByteBuffer bb : mmap(fc, (int) chunkSize)) { for (ByteBuffer bb : mmap(fc, (int) chunkSize)) {
workerPool.submit(() -> { workerPool.submit(() -> {
@ -200,8 +194,6 @@ public class CalculateAverage_jbachorik {
} }
workerPool.shutdown(); workerPool.shutdown();
workerPool.awaitTermination(1, TimeUnit.HOURS); workerPool.awaitTermination(1, TimeUnit.HOURS);
mergerPool.shutdown();
mergerPool.awaitTermination(1, TimeUnit.HOURS);
} }
finally { finally {
// System.out.println("Keys: " + map.size()); // System.out.println("Keys: " + map.size());
@ -209,21 +201,9 @@ public class CalculateAverage_jbachorik {
} }
} }
private static int longHash(long l, int h) { // unrolled FNV-1a 64 bit hash
if (l == 0) { private static final long fnv64OffsetBasis = 0xCBF29CE484222325L;
return h; private static final long fnv64Prime = 0x100000001B3L;
}
h = (int) (31 * 31 * 31 * 31 * 31 * 31 * 31 * 31 * h
+ 31 * 31 * 31 * 31 * 31 * 31 * 31 * ((l >> 56 & 0xFF))
+ 31 * 31 * 31 * 31 * 31 * 31 * ((l >> 48 & 0xFF))
+ 31 * 31 * 31 * 31 * 31 * ((l >> 40 & 0xFF))
+ 31 * 31 * 31 * 31 * ((l >> 32 & 0xFF))
+ 31 * 31 * 31 * ((l >> 24 & 0xFF))
+ 31 * 31 * ((l >> 16) & 0xFF)
+ 31 * ((l >> 8) & 0xFF)
+ (l & 0xFF));
return h;
}
private static StatsMap processChunk(ByteBuffer bb) { private static StatsMap processChunk(ByteBuffer bb) {
StatsMap map = new StatsMap(); StatsMap map = new StatsMap();
@ -233,7 +213,7 @@ public class CalculateAverage_jbachorik {
int readLimit = limit - 8; int readLimit = limit - 8;
long v0 = 0; long v0 = 0;
long v1 = 0; long v1 = 0;
int hashCode = 0; long hashCode = fnv64OffsetBasis;
int lastNewLine = -1; int lastNewLine = -1;
while (offset < limit) { while (offset < limit) {
@ -249,24 +229,34 @@ public class CalculateAverage_jbachorik {
if (x != 0) { if (x != 0) {
long value = 0; long value = 0;
int valueLen = 0; int valueLen = 0;
int pos = 7 - (Long.numberOfTrailingZeros(x) >>> 3); int pos = Long.numberOfLeadingZeros(x) >>> 3;
int yoffset = offset; int yoffset = offset;
int semiPos = firstInstance(v1, semiPattern); int semiPos = firstInstance(v1, semiPattern);
if (semiPos == 8 || semiPos >= pos) { if (semiPos >= pos) {
yoffset -= 8; yoffset -= 8;
semiPos = firstInstance(v0, semiPattern); semiPos = firstInstance(v0, semiPattern);
// semiPos will be at least 3 (new line is in the upper word and the value has at most 5 bytes) // semiPos will be at least 3 (new line is in the upper word and the value has at most 5 bytes)
long mask = semiPos == 0 ? 0L : (0xFFFFFFFFFFFFFFFFL << (8 - semiPos) * 8); // a 64 bit value can not be rotated by 64 bits to 'clear' the bits
long newlineMask = pos == 0 ? 0L : (0xFFFFFFFFFFFFFFFFL << ((8 - pos) * 8)); // instead, it must be rotated by at most 56 bits and then, if 64 bit rotation was requested, by 8 bits more
int rot2 = (8 - semiPos) >>> 3;
int rot1 = (7 - semiPos) + (~rot2 & 0x1);
long mask = ((0xFFFFFFFFFFFFFFFFL << (rot1 * 8)) << (rot2 * 8));
rot2 = (8 - pos) >>> 3;
rot1 = (7 - pos) + (~rot2 & 0x1);
long newlineMask = ((0xFFFFFFFFFFFFFFFFL << (rot1 * 8)) << (rot2 * 8));
value = semiPos == 7 ? 0L : (v0 << (semiPos + 1) * 8); value = semiPos == 7 ? 0L : (v0 << (semiPos + 1) * 8);
value |= ((v1 & newlineMask) >> (7 - semiPos) * 8); value |= ((v1 & newlineMask) >> (7 - semiPos) * 8);
// right-align the value bytes
// getting the number of trailing zeros is the easiest way to figure out the shift
// should be sufficiently fast but ...
int zeros = (Long.numberOfTrailingZeros(value) >>> 3); int zeros = (Long.numberOfTrailingZeros(value) >>> 3);
value = value >>> zeros * 8; value = value >>> zeros * 8;
valueLen = 8 - zeros; valueLen = 8 - zeros;
v0 = v0 & mask; v0 = v0 & mask;
} }
else { else {
hashCode += longHash(v0, hashCode); hashCode ^= v0;
hashCode *= fnv64Prime;
long valMask = (0xFFFFFFFFFFFFFFFFL << (7 - semiPos) * 8); long valMask = (0xFFFFFFFFFFFFFFFFL << (7 - semiPos) * 8);
v0 = v1 & valMask; v0 = v1 & valMask;
value = v1 & ~valMask; value = v1 & ~valMask;
@ -274,26 +264,33 @@ public class CalculateAverage_jbachorik {
valueLen = pos - semiPos - 1; valueLen = pos - semiPos - 1;
} }
v1 = 0; v1 = 0;
hashCode += longHash(v0, hashCode); hashCode ^= v0;
hashCode *= fnv64Prime;
int len = (yoffset + semiPos - 1) - lastNewLine; int len = (yoffset + semiPos - 1) - lastNewLine;
hashCode += longHash(len, hashCode); hashCode ^= len;
hashCode *= fnv64Prime;
// byte[] strBuf = new byte[len]; // byte[] strBuf = new byte[len];
// bb.get(lastNewLine + 1, strBuf); // bb.get(lastNewLine + 1, strBuf);
// String str = new String(strBuf); // String str = new String(strBuf);
// System.out.println("===> " + str + ": " + Long.toHexString(value) + " :: " + fastParse(value, valueLen)); // System.out.println("===> " + str + ": " + Long.toHexString(value) + " :: " + fastParse(value, valueLen));
map.getOrInsert(bb, lastNewLine + 1, len, hashCode, v0, v1).add(fastParse(value, valueLen)); // projection of the hash code to 32 bits -> 65k buckets
long idx = ((hashCode & 0xFFFFFFFF00000000L) >> 32) ^ (hashCode & 0x00000000FFFFFFFFL);
idx = ((idx & 0x00000000FFFF0000L) >> 16) ^ (idx & 0x000000000000FFFFL);
map.getOrInsert(bb, lastNewLine + 1, len, (int) idx, v0, v1).add(fastParse(value, valueLen));
offset += pos + 1; offset += pos + 1;
lastNewLine = offset - 1; lastNewLine = offset - 1;
// reset the previous value // reset the previous value
v0 = 0; v0 = 0;
// reset the hash // reset the hash
hashCode = 0; hashCode = fnv64OffsetBasis;
} }
else { else {
offset += 8; offset += 8;
hashCode += longHash(v0, hashCode); hashCode ^= v0;
hashCode *= fnv64Prime;
v0 = v1; v0 = v1;
} }
} }
@ -305,10 +302,12 @@ public class CalculateAverage_jbachorik {
private static int fastParse(long word, int len) { private static int fastParse(long word, int len) {
assert (len <= 5); assert (len <= 5);
long singChar = (word >> ((len - 1) * 8)); int signChar = (int) (word >> ((len - 1) * 8)) & 0xFF;
long sign = singChar & 0x2dL; int sign = signChar ^ 0x2d;
int negative = (int) ((sign ^ 0x2d) & 0xff) == 0 ? -1 : 1; int base = ~(sign | -sign);
int shift = (8 - len - Math.min(negative, 0)) * 8; int offset = (base >> 7) & 0x01;
int multiplier = -(~((sign - 1) >> 31) | 0x1);
int shift = (8 - len + offset) * 8;
long mask = (0xFFFFFFFFFFFFFFFFL >>> shift); long mask = (0xFFFFFFFFFFFFFFFFL >>> shift);
word = (word ^ fastParserMask) & mask; word = (word ^ fastParserMask) & mask;
@ -319,8 +318,7 @@ public class CalculateAverage_jbachorik {
int v4 = 100 * ((int) (word >> 24) & 0xff); int v4 = 100 * ((int) (word >> 24) & 0xff);
// v5 is either the sign or not used // v5 is either the sign or not used
int val = ((v1 + v3 + v4) * negative); return ((v1 + v3 + v4) * multiplier);
return val;
} }
private static ByteBuffer[] mmap(FileChannel fc, int splitSize) throws Exception { private static ByteBuffer[] mmap(FileChannel fc, int splitSize) throws Exception {