Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

jerrinot - final(?) improvements (#690)

Browse Source 
* decrease instruction level parallelism

it turns out doing 2 things was too much. perf annotate showed spilling.

* more trickery with latency hiding

* work-stealing, lookp tables, credits

* do not assume gender
This commit is contained in:
Jaromir Hamala 2024-01-31 17:17:34 +00:00 committed by GitHub
parent b529ef2a59
commit 9b9bb8ed3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 182 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

347
src/main/java/dev/morling/onebrc/CalculateAverage_jerrinot.java
View File

@ -24,6 +24,7 @@ import java.lang.foreign.Arena;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.nio.channels.FileChannel.MapMode; import java.nio.channels.FileChannel.MapMode;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
/** /**
* I figured out it would be very hard to win the main competition of the One Billion Rows Challenge. * I figured out it would be very hard to win the main competition of the One Billion Rows Challenge.
@ -31,17 +32,59 @@ import java.util.*;
* *
* Anyway, if you can make sense out of not exactly idiomatic Java code, and you enjoy pushing performance limits * Anyway, if you can make sense out of not exactly idiomatic Java code, and you enjoy pushing performance limits
* then QuestDB - the fastest open-source time-series database - is hiring: https://questdb.io/careers/core-database-engineer/ * then QuestDB - the fastest open-source time-series database - is hiring: https://questdb.io/careers/core-database-engineer/
* * <p>
* <b>Credit</b>
* <p>
* I stand on shoulders of giants. I wouldn't be able to code this without analyzing and borrowing from solutions of others.
* People who helped me the most:
* <ul>
* <li>Thomas Wuerthinger (thomaswue): The munmap() trick and work-stealing. In both cases, I shameless copy-pasted their code.
* Including SWAR for detecting new lines. Thomas also gave me helpful hints on how to detect register spilling issues.</li>
* <li>Quan Anh Mai (merykitty): I borrowed their phenomenal branch-free parser.</li>
* <li>Marko Topolnik (mtopolnik): I use a hashing function I saw in his code. It seems the produce good quality hashes
* and it's next-level in speed. Marko joined the challenge before me and our discussions made me to join too!</li>
* <li>Van Phu DO (abeobk): I saw the idea with simple lookup tables instead of complicated bit-twiddling in their code first.</li>
* <li>Roy van Rijn (royvanrijn): I borrowed their SWAR code and initially their hash code impl</li>
* <li>Francesco Nigro (franz1981): For our online discussions about performance. Both before and during this challenge.
* Francesco gave me the idea to check register spilling.</li>
* </ul>
*/ */
public class CalculateAverage_jerrinot { public class CalculateAverage_jerrinot {
private static final Unsafe UNSAFE = unsafe(); private static final Unsafe UNSAFE = unsafe();
private static final String MEASUREMENTS_TXT = "measurements.txt"; private static final String MEASUREMENTS_TXT = "measurements.txt";
// todo: with hyper-threading enable we would be better of with availableProcessors / 2; // todo: with hyper-threading enable we would be better of with availableProcessors / 2;
// todo: validate the testing env. params. // todo: validate the testing env. params.
private static final int THREAD_COUNT = Runtime.getRuntime().availableProcessors(); private static final int EXTRA_THREAD_COUNT = Runtime.getRuntime().availableProcessors() - 1;
// private static final int THREAD_COUNT = 4; // private static final int THREAD_COUNT = 1;
private static final long SEPARATOR_PATTERN = 0x3B3B3B3B3B3B3B3BL; private static final long SEPARATOR_PATTERN = 0x3B3B3B3B3B3B3B3BL;
private static final long NEW_LINE_PATTERN = 0x0A0A0A0A0A0A0A0AL;
private static final int SEGMENT_SIZE = 4 * 1024 * 1024;
// credits for the idea with lookup tables instead of bit-shifting: abeobk
private static final long[] HASH_MASKS = new long[]{
0x0000000000000000L, // semicolon is the first char
0x00000000000000ffL,
0x000000000000ffffL,
0x0000000000ffffffL,
0x00000000ffffffffL,
0x000000ffffffffffL,
0x0000ffffffffffffL,
0x00ffffffffffffffL, // semicolon is the last char
0xffffffffffffffffL // there is no semicolon at all
};
private static final long[] ADVANCE_MASKS = new long[]{
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0x0000000000000000L,
0xffffffffffffffffL,
};
private static Unsafe unsafe() { private static Unsafe unsafe() {
try { try {
@ -81,56 +124,29 @@ public class CalculateAverage_jerrinot {
static void calculate() throws Exception { static void calculate() throws Exception {
final File file = new File(MEASUREMENTS_TXT); final File file = new File(MEASUREMENTS_TXT);
final long length = file.length(); final long length = file.length();
// final int chunkCount = Runtime.getRuntime().availableProcessors();
int chunkPerThread = 3;
final int chunkCount = THREAD_COUNT * chunkPerThread;
final var chunkStartOffsets = new long[chunkCount + 1];
try (var raf = new RandomAccessFile(file, "r")) { try (var raf = new RandomAccessFile(file, "r")) {
// credit - chunking code: mtopolnik long fileStart = raf.getChannel().map(MapMode.READ_ONLY, 0, length, Arena.global()).address();
final var inputBase = raf.getChannel().map(MapMode.READ_ONLY, 0, length, Arena.global()).address(); long fileEnd = fileStart + length;
for (int i = 1; i < chunkStartOffsets.length - 1; i++) { var globalCursor = new AtomicLong(fileStart);
var start = length * i / (chunkStartOffsets.length - 1);
raf.seek(start);
while (raf.read() != (byte) '\n') {
}
start = raf.getFilePointer();
chunkStartOffsets[i] = start + inputBase;
}
chunkStartOffsets[0] = inputBase;
chunkStartOffsets[chunkCount] = inputBase + length;
Processor[] processors = new Processor[THREAD_COUNT]; Processor[] processors = new Processor[EXTRA_THREAD_COUNT];
Thread[] threads = new Thread[THREAD_COUNT]; Thread[] threads = new Thread[EXTRA_THREAD_COUNT];
for (int i = 0; i < THREAD_COUNT - 1; i++) { for (int i = 0; i < EXTRA_THREAD_COUNT; i++) {
long startA = chunkStartOffsets[i * chunkPerThread]; Processor processor = new Processor(fileStart, fileEnd, globalCursor);
long endA = chunkStartOffsets[i * chunkPerThread + 1];
long startB = chunkStartOffsets[i * chunkPerThread + 1];
long endB = chunkStartOffsets[i * chunkPerThread + 2];
long startC = chunkStartOffsets[i * chunkPerThread + 2];
long endC = chunkStartOffsets[i * chunkPerThread + 3];
Processor processor = new Processor(startA, endA, startB, endB, startC, endC);
processors[i] = processor;
Thread thread = new Thread(processor); Thread thread = new Thread(processor);
processors[i] = processor;
threads[i] = thread; threads[i] = thread;
thread.start(); thread.start();
} }
int ownIndex = THREAD_COUNT - 1; Processor processor = new Processor(fileStart, fileEnd, globalCursor);
long startA = chunkStartOffsets[ownIndex * chunkPerThread];
long endA = chunkStartOffsets[ownIndex * chunkPerThread + 1];
long startB = chunkStartOffsets[ownIndex * chunkPerThread + 1];
long endB = chunkStartOffsets[ownIndex * chunkPerThread + 2];
long startC = chunkStartOffsets[ownIndex * chunkPerThread + 2];
long endC = chunkStartOffsets[ownIndex * chunkPerThread + 3];
Processor processor = new Processor(startA, endA, startB, endB, startC, endC);
processor.run(); processor.run();
var accumulator = new TreeMap<String, Processor.StationStats>(); var accumulator = new TreeMap<String, StationStats>();
processor.accumulateStatus(accumulator); processor.accumulateStatus(accumulator);
for (int i = 0; i < THREAD_COUNT - 1; i++) { for (int i = 0; i < EXTRA_THREAD_COUNT; i++) {
Thread t = threads[i]; Thread t = threads[i];
t.join(); t.join();
processors[i].accumulateStatus(accumulator); processors[i].accumulateStatus(accumulator);
@ -140,10 +156,10 @@ public class CalculateAverage_jerrinot {
} }
} }
private static void printResults(TreeMap<String, Processor.StationStats> accumulator) { private static void printResults(TreeMap<String, StationStats> accumulator) {
var sb = new StringBuilder(10000); var sb = new StringBuilder(10000);
boolean first = true; boolean first = true;
for (Map.Entry<String, Processor.StationStats> statsEntry : accumulator.entrySet()) { for (Map.Entry<String, StationStats> statsEntry : accumulator.entrySet()) {
if (first) { if (first) {
sb.append("{"); sb.append("{");
first = false; first = false;
@ -210,20 +226,17 @@ public class CalculateAverage_jerrinot {
private static final int FAST_MAP_SIZE_BYTES = MAPS_SLOT_COUNT * FAST_MAP_ENTRY_SIZE_BYTES; private static final int FAST_MAP_SIZE_BYTES = MAPS_SLOT_COUNT * FAST_MAP_ENTRY_SIZE_BYTES;
private static final int SLOW_MAP_MAP_NAMES_BYTES = MAX_UNIQUE_KEYS * STATION_MAX_NAME_BYTES; private static final int SLOW_MAP_MAP_NAMES_BYTES = MAX_UNIQUE_KEYS * STATION_MAX_NAME_BYTES;
private static final int MAP_MASK = MAPS_SLOT_COUNT - 1; private static final int MAP_MASK = MAPS_SLOT_COUNT - 1;
private final AtomicLong globalCursor;
private long slowMap; private long slowMap;
private long slowMapNamesPtr; private long slowMapNamesPtr;
private long slowMapNamesLo;
// private long fastMap;
private long cursorA; private long cursorA;
private long endA; private long endA;
private long cursorB; private long cursorB;
private long endB; private long endB;
private long cursorC; private HashMap<String, CalculateAverage_jerrinot.StationStats> stats = new HashMap<>(1000);
private long endC; private final long fileEnd;
private HashMap<String, StationStats> stats = new HashMap<>(1000); private final long fileStart;
// private long maxClusterLen;
// credit: merykitty // credit: merykitty
private long parseAndStoreTemperature(long startCursor, long baseEntryPtr, long word) { private long parseAndStoreTemperature(long startCursor, long baseEntryPtr, long word) {
@ -264,20 +277,12 @@ public class CalculateAverage_jerrinot {
return (match - 0x0101010101010101L) & (~match & 0x8080808080808080L); return (match - 0x0101010101010101L) & (~match & 0x8080808080808080L);
} }
// todo: immutability cost us in allocations, but that's probably peanuts in the grand scheme of things. still worth checking void accumulateStatus(TreeMap<String, CalculateAverage_jerrinot.StationStats> accumulator) {
// maybe JVM trusting Final in Records offsets it ..a test is needed for (Map.Entry<String, CalculateAverage_jerrinot.StationStats> entry : stats.entrySet()) {
record StationStats(int min, int max, int count, long sum) {
StationStats mergeWith(StationStats other) {
return new StationStats(Math.min(min, other.min), Math.max(max, other.max), count + other.count, sum + other.sum);
}
}
void accumulateStatus(TreeMap<String, StationStats> accumulator) {
for (Map.Entry<String, StationStats> entry : stats.entrySet()) {
String name = entry.getKey(); String name = entry.getKey();
StationStats localStats = entry.getValue(); CalculateAverage_jerrinot.StationStats localStats = entry.getValue();
StationStats globalStats = accumulator.get(name); CalculateAverage_jerrinot.StationStats globalStats = accumulator.get(name);
if (globalStats == null) { if (globalStats == null) {
accumulator.put(name, localStats); accumulator.put(name, localStats);
} }
@ -287,24 +292,10 @@ public class CalculateAverage_jerrinot {
} }
} }
Processor(long startA, long endA, long startB, long endB, long startC, long endC) { Processor(long fileStart, long fileEnd, AtomicLong globalCursor) {
this.cursorA = startA; this.globalCursor = globalCursor;
this.cursorB = startB; this.fileEnd = fileEnd;
this.cursorC = startC; this.fileStart = fileStart;
this.endA = endA;
this.endB = endB;
this.endC = endC;
}
private void doTail(long fastMAp) {
doOne(cursorA, endA);
doOne(cursorB, endB);
doOne(cursorC, endC);
transferToHeap(fastMAp);
// UNSAFE.freeMemory(fastMap);
// UNSAFE.freeMemory(slowMap);
// UNSAFE.freeMemory(slowMapNamesLo);
} }
private void transferToHeap(long fastMap) { private void transferToHeap(long fastMap) {
@ -324,7 +315,7 @@ public class CalculateAverage_jerrinot {
int count = UNSAFE.getInt(baseAddress + MAP_COUNT_OFFSET); int count = UNSAFE.getInt(baseAddress + MAP_COUNT_OFFSET);
long sum = UNSAFE.getLong(baseAddress + MAP_SUM_OFFSET); long sum = UNSAFE.getLong(baseAddress + MAP_SUM_OFFSET);
stats.put(name, new StationStats(min, max, count, sum)); stats.put(name, new CalculateAverage_jerrinot.StationStats(min, max, count, sum));
} }
for (long baseAddress = fastMap; baseAddress < fastMap + FAST_MAP_SIZE_BYTES; baseAddress += FAST_MAP_ENTRY_SIZE_BYTES) { for (long baseAddress = fastMap; baseAddress < fastMap + FAST_MAP_SIZE_BYTES; baseAddress += FAST_MAP_ENTRY_SIZE_BYTES) {
@ -345,16 +336,21 @@ public class CalculateAverage_jerrinot {
var v = stats.get(name); var v = stats.get(name);
if (v == null) { if (v == null) {
stats.put(name, new StationStats(min, max, count, sum)); stats.put(name, new CalculateAverage_jerrinot.StationStats(min, max, count, sum));
} }
else { else {
stats.put(name, new StationStats(Math.min(v.min, min), Math.max(v.max, max), v.count + count, v.sum + sum)); stats.put(name, new CalculateAverage_jerrinot.StationStats(Math.min(v.min, min), Math.max(v.max, max), v.count + count, v.sum + sum));
} }
} }
} }
private void doOne(long cursor, long endA) { private void doOne(long cursor, long end) {
while (cursor < endA) { while (cursor < end) {
// it seems that when pulling just from a single chunk
// then bit-twiddling is faster than lookup tables
// hypothesis: when processing multiple things at once then LOAD latency is partially hidden
// but when processing just one thing then it's better to keep things local as much as possible? maybe:)
long start = cursor; long start = cursor;
long currentWord = UNSAFE.getLong(cursor); long currentWord = UNSAFE.getLong(cursor);
long mask = getDelimiterMask(currentWord); long mask = getDelimiterMask(currentWord);
@ -392,135 +388,139 @@ public class CalculateAverage_jerrinot {
return (int) hash; return (int) hash;
} }
private static long nextNewLine(long prev) {
// again: credits to @thomaswue for this code, literally copy'n'paste
while (true) {
long currentWord = UNSAFE.getLong(prev);
long input = currentWord ^ NEW_LINE_PATTERN;
long pos = (input - 0x0101010101010101L) & ~input & 0x8080808080808080L;
if (pos != 0) {
prev += Long.numberOfTrailingZeros(pos) >>> 3;
break;
}
else {
prev += 8;
}
}
return prev;
}
@Override @Override
public void run() { public void run() {
long fastMap = allocateMem();
for (;;) {
long startingPtr = globalCursor.addAndGet(SEGMENT_SIZE) - SEGMENT_SIZE;
if (startingPtr >= fileEnd) {
break;
}
setCursors(startingPtr);
mainLoop(fastMap);
doOne(cursorA, endA);
doOne(cursorB, endB);
}
transferToHeap(fastMap);
}
private long allocateMem() {
this.slowMap = UNSAFE.allocateMemory(SLOW_MAP_SIZE_BYTES); this.slowMap = UNSAFE.allocateMemory(SLOW_MAP_SIZE_BYTES);
this.slowMapNamesPtr = UNSAFE.allocateMemory(SLOW_MAP_MAP_NAMES_BYTES); this.slowMapNamesPtr = UNSAFE.allocateMemory(SLOW_MAP_MAP_NAMES_BYTES);
this.slowMapNamesLo = slowMapNamesPtr;
long fastMap = UNSAFE.allocateMemory(FAST_MAP_SIZE_BYTES); long fastMap = UNSAFE.allocateMemory(FAST_MAP_SIZE_BYTES);
UNSAFE.setMemory(slowMap, SLOW_MAP_SIZE_BYTES, (byte) 0); UNSAFE.setMemory(slowMap, SLOW_MAP_SIZE_BYTES, (byte) 0);
UNSAFE.setMemory(fastMap, FAST_MAP_SIZE_BYTES, (byte) 0); UNSAFE.setMemory(fastMap, FAST_MAP_SIZE_BYTES, (byte) 0);
UNSAFE.setMemory(slowMapNamesPtr, SLOW_MAP_MAP_NAMES_BYTES, (byte) 0); UNSAFE.setMemory(slowMapNamesPtr, SLOW_MAP_MAP_NAMES_BYTES, (byte) 0);
return fastMap;
}
while (cursorA < endA && cursorB < endB && cursorC < endC) { private void mainLoop(long fastMap) {
while (cursorA < endA && cursorB < endB) {
long currentWordA = UNSAFE.getLong(cursorA); long currentWordA = UNSAFE.getLong(cursorA);
long currentWordB = UNSAFE.getLong(cursorB); long currentWordB = UNSAFE.getLong(cursorB);
long currentWordC = UNSAFE.getLong(cursorC);
long delimiterMaskA = getDelimiterMask(currentWordA);
long delimiterMaskB = getDelimiterMask(currentWordB);
long candidateWordA = UNSAFE.getLong(cursorA + 8);
long candidateWordB = UNSAFE.getLong(cursorB + 8);
long startA = cursorA; long startA = cursorA;
long startB = cursorB; long startB = cursorB;
long startC = cursorC;
long maskA = getDelimiterMask(currentWordA); int trailingZerosA = Long.numberOfTrailingZeros(delimiterMaskA) >> 3;
long maskB = getDelimiterMask(currentWordB); int trailingZerosB = Long.numberOfTrailingZeros(delimiterMaskB) >> 3;
long maskC = getDelimiterMask(currentWordC);
long maskComplementA = -maskA; long advanceMaskA = ADVANCE_MASKS[trailingZerosA];
long maskComplementB = -maskB; long advanceMaskB = ADVANCE_MASKS[trailingZerosB];
long maskComplementC = -maskC;
long maskWithDelimiterA = (maskA ^ (maskA - 1)); long wordMaskA = HASH_MASKS[trailingZerosA];
long maskWithDelimiterB = (maskB ^ (maskB - 1)); long wordMaskB = HASH_MASKS[trailingZerosB];
long maskWithDelimiterC = (maskC ^ (maskC - 1));
long isMaskZeroA = (((maskA | maskComplementA) >>> 63) ^ 1); long negAdvanceMaskA = ~advanceMaskA;
long isMaskZeroB = (((maskB | maskComplementB) >>> 63) ^ 1); long negAdvanceMaskB = ~advanceMaskB;
long isMaskZeroC = (((maskC | maskComplementC) >>> 63) ^ 1);
cursorA += isMaskZeroA << 3; cursorA += advanceMaskA & 8;
cursorB += isMaskZeroB << 3; cursorB += advanceMaskB & 8;
cursorC += isMaskZeroC << 3;
long nextWordA = UNSAFE.getLong(cursorA); long nextWordA = (advanceMaskA & candidateWordA) | (negAdvanceMaskA & currentWordA);
long nextWordB = UNSAFE.getLong(cursorB); long nextWordB = (advanceMaskB & candidateWordB) | (negAdvanceMaskB & currentWordB);
long nextWordC = UNSAFE.getLong(cursorC);
long firstWordMaskA = maskWithDelimiterA >>> 8; long nextDelimiterMaskA = getDelimiterMask(nextWordA);
long firstWordMaskB = maskWithDelimiterB >>> 8; long nextDelimiterMaskB = getDelimiterMask(nextWordB);
long firstWordMaskC = maskWithDelimiterC >>> 8;
long nextMaskA = getDelimiterMask(nextWordA); boolean slowA = nextDelimiterMaskA == 0;
long nextMaskB = getDelimiterMask(nextWordB); boolean slowB = nextDelimiterMaskB == 0;
long nextMaskC = getDelimiterMask(nextWordC); boolean slowSome = (slowA || slowB);
boolean slowA = nextMaskA == 0; long maskedFirstWordA = wordMaskA & currentWordA;
boolean slowB = nextMaskB == 0; long maskedFirstWordB = wordMaskB & currentWordB;
boolean slowC = nextMaskC == 0;
boolean slowSome = (slowA || slowB || slowC);
long extA = -isMaskZeroA;
long extB = -isMaskZeroB;
long extC = -isMaskZeroC;
long maskedFirstWordA = (extA | firstWordMaskA) & currentWordA;
long maskedFirstWordB = (extB | firstWordMaskB) & currentWordB;
long maskedFirstWordC = (extC | firstWordMaskC) & currentWordC;
int hashA = hash(maskedFirstWordA); int hashA = hash(maskedFirstWordA);
int hashB = hash(maskedFirstWordB); int hashB = hash(maskedFirstWordB);
int hashC = hash(maskedFirstWordC);
currentWordA = nextWordA; currentWordA = nextWordA;
currentWordB = nextWordB; currentWordB = nextWordB;
currentWordC = nextWordC;
maskA = nextMaskA; delimiterMaskA = nextDelimiterMaskA;
maskB = nextMaskB; delimiterMaskB = nextDelimiterMaskB;
maskC = nextMaskC;
if (slowSome) { if (slowSome) {
while (maskA == 0) { while (delimiterMaskA == 0) {
cursorA += 8; cursorA += 8;
currentWordA = UNSAFE.getLong(cursorA); currentWordA = UNSAFE.getLong(cursorA);
maskA = getDelimiterMask(currentWordA); delimiterMaskA = getDelimiterMask(currentWordA);
} }
while (maskB == 0) { while (delimiterMaskB == 0) {
cursorB += 8; cursorB += 8;
currentWordB = UNSAFE.getLong(cursorB); currentWordB = UNSAFE.getLong(cursorB);
maskB = getDelimiterMask(currentWordB); delimiterMaskB = getDelimiterMask(currentWordB);
}
while (maskC == 0) {
cursorC += 8;
currentWordC = UNSAFE.getLong(cursorC);
maskC = getDelimiterMask(currentWordC);
} }
} }
final int delimiterByteA = Long.numberOfTrailingZeros(maskA); trailingZerosA = Long.numberOfTrailingZeros(delimiterMaskA) >> 3;
final int delimiterByteB = Long.numberOfTrailingZeros(maskB); trailingZerosB = Long.numberOfTrailingZeros(delimiterMaskB) >> 3;
final int delimiterByteC = Long.numberOfTrailingZeros(maskC);
final long semicolonA = cursorA + (delimiterByteA >> 3); final long semicolonA = cursorA + trailingZerosA;
final long semicolonB = cursorB + (delimiterByteB >> 3); final long semicolonB = cursorB + trailingZerosB;
final long semicolonC = cursorC + (delimiterByteC >> 3);
long digitStartA = semicolonA + 1; long digitStartA = semicolonA + 1;
long digitStartB = semicolonB + 1; long digitStartB = semicolonB + 1;
long digitStartC = semicolonC + 1;
long lastWordMaskA = HASH_MASKS[trailingZerosA];
long lastWordMaskB = HASH_MASKS[trailingZerosB];
long temperatureWordA = UNSAFE.getLong(digitStartA); long temperatureWordA = UNSAFE.getLong(digitStartA);
long temperatureWordB = UNSAFE.getLong(digitStartB); long temperatureWordB = UNSAFE.getLong(digitStartB);
long temperatureWordC = UNSAFE.getLong(digitStartC);
long lastWordMaskA = ((maskA - 1) ^ maskA) >>> 8;
long lastWordMaskB = ((maskB - 1) ^ maskB) >>> 8;
long lastWordMaskC = ((maskC - 1) ^ maskC) >>> 8;
final long maskedLastWordA = currentWordA & lastWordMaskA; final long maskedLastWordA = currentWordA & lastWordMaskA;
final long maskedLastWordB = currentWordB & lastWordMaskB; final long maskedLastWordB = currentWordB & lastWordMaskB;
final long maskedLastWordC = currentWordC & lastWordMaskC;
int lenA = (int) (semicolonA - startA); int lenA = (int) (semicolonA - startA);
int lenB = (int) (semicolonB - startB); int lenB = (int) (semicolonB - startB);
int lenC = (int) (semicolonC - startC);
int mapIndexA = hashA & MAP_MASK; int mapIndexA = hashA & MAP_MASK;
int mapIndexB = hashB & MAP_MASK; int mapIndexB = hashB & MAP_MASK;
int mapIndexC = hashC & MAP_MASK;
long baseEntryPtrA; long baseEntryPtrA;
long baseEntryPtrB; long baseEntryPtrB;
long baseEntryPtrC;
if (slowSome) { if (slowSome) {
if (slowA) { if (slowA) {
@ -537,25 +537,37 @@ public class CalculateAverage_jerrinot {
baseEntryPtrB = getOrCreateEntryBaseOffsetFast(mapIndexB, lenB, maskedLastWordB, maskedFirstWordB, fastMap); baseEntryPtrB = getOrCreateEntryBaseOffsetFast(mapIndexB, lenB, maskedLastWordB, maskedFirstWordB, fastMap);
} }
if (slowC) {
baseEntryPtrC = getOrCreateEntryBaseOffsetSlow(lenC, startC, hashC, maskedLastWordC);
}
else {
baseEntryPtrC = getOrCreateEntryBaseOffsetFast(mapIndexC, lenC, maskedLastWordC, maskedFirstWordC, fastMap);
}
} }
else { else {
baseEntryPtrA = getOrCreateEntryBaseOffsetFast(mapIndexA, lenA, maskedLastWordA, maskedFirstWordA, fastMap); baseEntryPtrA = getOrCreateEntryBaseOffsetFast(mapIndexA, lenA, maskedLastWordA, maskedFirstWordA, fastMap);
baseEntryPtrB = getOrCreateEntryBaseOffsetFast(mapIndexB, lenB, maskedLastWordB, maskedFirstWordB, fastMap); baseEntryPtrB = getOrCreateEntryBaseOffsetFast(mapIndexB, lenB, maskedLastWordB, maskedFirstWordB, fastMap);
baseEntryPtrC = getOrCreateEntryBaseOffsetFast(mapIndexC, lenC, maskedLastWordC, maskedFirstWordC, fastMap);
} }
cursorA = parseAndStoreTemperature(digitStartA, baseEntryPtrA, temperatureWordA); cursorA = parseAndStoreTemperature(digitStartA, baseEntryPtrA, temperatureWordA);
cursorB = parseAndStoreTemperature(digitStartB, baseEntryPtrB, temperatureWordB); cursorB = parseAndStoreTemperature(digitStartB, baseEntryPtrB, temperatureWordB);
cursorC = parseAndStoreTemperature(digitStartC, baseEntryPtrC, temperatureWordC);
} }
doTail(fastMap); }
// System.out.println("Longest chain: " + longestChain);
private void setCursors(long current) {
// Credit for the whole work-stealing scheme: @thomaswue
// I have totally stolen it from him. I changed the order a bit to suite my taste better,
// but it's his code
long segmentStart;
if (current == fileStart) {
segmentStart = current;
}
else {
segmentStart = nextNewLine(current) + 1;
}
long segmentEnd = nextNewLine(Math.min(fileEnd - 1, current + SEGMENT_SIZE));
long size = (segmentEnd - segmentStart) / 2;
long mid = nextNewLine(segmentStart + size);
cursorA = segmentStart;
endA = mid;
cursorB = mid + 1;
endB = segmentEnd;
} }
private static long getOrCreateEntryBaseOffsetFast(int mapIndexA, int lenA, long maskedLastWord, long maskedFirstWord, long fastMap) { private static long getOrCreateEntryBaseOffsetFast(int mapIndexA, int lenA, long maskedLastWord, long maskedFirstWord, long fastMap) {
@ -625,4 +637,9 @@ public class CalculateAverage_jerrinot {
} }
} }
record StationStats(int min, int max, int count, long sum) {
StationStats mergeWith(StationStats other) {
return new StationStats(Math.min(min, other.min), Math.max(max, other.max), count + other.count, sum + other.sum);
}
}
} }