Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

One HashMap per thread: 23" to 18" locally but hopefully more on the server (#319)

Browse Source
This commit is contained in:
Anthony Goubard 2024-01-11 21:11:08 +01:00 committed by GitHub
parent 8ef8cd2b17
commit 11a740c5d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/dev/morling/onebrc/CalculateAverage_japplis.java
View File

@ -40,6 +40,8 @@ import java.util.concurrent.*;
* - Replaced String with my own ByteText class: a bit slower (~10%) * - Replaced String with my own ByteText class: a bit slower (~10%)
* - Replaced compute lambda call with synchronized(city.intern()): 43" (due to intern()) * - Replaced compute lambda call with synchronized(city.intern()): 43" (due to intern())
* - Removed BufferedInputStream and replaced Measurement with IntSummaryStatistics (thanks davecom): still 23" but cleaner code * - Removed BufferedInputStream and replaced Measurement with IntSummaryStatistics (thanks davecom): still 23" but cleaner code
* - Execute same code on 1BRC server: 41"
* - One HashMap per thread: 17" locally
* *
* @author Anthony Goubard - Japplis * @author Anthony Goubard - Japplis
*/ */
@ -83,14 +85,16 @@ public class CalculateAverage_japplis {
int startIndex = handleSplitLine(buffer, readCount); int startIndex = handleSplitLine(buffer, readCount);
Runnable countAverageRun = () -> { Runnable countAverageRun = () -> {
int bufferIndex = startIndex; int bufferIndex = startIndex;
Map<String, IntSummaryStatistics> blockCityMeasurementMap = new HashMap<>();
try { try {
while (bufferIndex < readCount) { while (bufferIndex < readCount) {
bufferIndex = readNextLine(bufferIndex, buffer); bufferIndex = readNextLine(bufferIndex, buffer, blockCityMeasurementMap);
} }
} }
catch (ArrayIndexOutOfBoundsException ex) { catch (ArrayIndexOutOfBoundsException ex) {
// Done reading and parsing the buffer // Done reading and parsing the buffer
} }
mergeBlockResults(blockCityMeasurementMap);
readFileLock.release(); readFileLock.release();
}; };
return countAverageRun; return countAverageRun;
@ -128,7 +132,7 @@ public class CalculateAverage_japplis {
for (int i = 0; i < splitLineBytes.length; i++) { for (int i = 0; i < splitLineBytes.length; i++) {
splitLineBytes[i] = previousBlockLastLine.get(i); splitLineBytes[i] = previousBlockLastLine.get(i);
} }
readNextLine(0, splitLineBytes); readNextLine(0, splitLineBytes, cityMeasurementMap);
return bufferIndex; return bufferIndex;
} }
@ -154,7 +158,7 @@ public class CalculateAverage_japplis {
return startIndex; return startIndex;
} }
private int readNextLine(int bufferIndex, byte[] buffer) { private int readNextLine(int bufferIndex, byte[] buffer, Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
int startLineIndex = bufferIndex; int startLineIndex = bufferIndex;
while (buffer[bufferIndex] != ';') while (buffer[bufferIndex] != ';')
bufferIndex++; bufferIndex++;
@ -166,7 +170,7 @@ public class CalculateAverage_japplis {
bufferIndex++; bufferIndex++;
if (temperature <= -precisionLimitTenth || temperature >= precisionLimitTenth) if (temperature <= -precisionLimitTenth || temperature >= precisionLimitTenth)
bufferIndex++; bufferIndex++;
addTemperature(city, temperature); addTemperature(city, temperature, blockCityMeasurementMap);
return bufferIndex; return bufferIndex;
} }
@ -187,12 +191,20 @@ public class CalculateAverage_japplis {
return temperature; return temperature;
} }
private void addTemperature(String city, int temperature) { private void addTemperature(String city, int temperature, Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
cityMeasurementMap.compute(city, (town, measurement) -> { IntSummaryStatistics measurement = blockCityMeasurementMap.get(city);
if (measurement == null) if (measurement == null) {
measurement = new IntSummaryStatistics(); measurement = new IntSummaryStatistics();
measurement.accept(temperature); blockCityMeasurementMap.put(city, measurement);
return measurement; }
measurement.accept(temperature);
}
private void mergeBlockResults(Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
blockCityMeasurementMap.forEach((city, measurement) -> {
IntSummaryStatistics oldMeasurement = cityMeasurementMap.putIfAbsent(city, measurement);
if (oldMeasurement != null)
oldMeasurement.combine(measurement);
}); });
} }