Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

One HashMap per thread: 23" to 18" locally but hopefully more on the server (#319)

Browse Source
This commit is contained in:
Anthony Goubard 2024-01-11 21:11:08 +01:00 committed by GitHub
parent 8ef8cd2b17
commit 11a740c5d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/dev/morling/onebrc/CalculateAverage_japplis.java
View File

@ -40,6 +40,8 @@ import java.util.concurrent.*;
* - Replaced String with my own ByteText class: a bit slower (~10%)
* - Replaced compute lambda call with synchronized(city.intern()): 43" (due to intern())
* - Removed BufferedInputStream and replaced Measurement with IntSummaryStatistics (thanks davecom): still 23" but cleaner code
* - Execute same code on 1BRC server: 41"
* - One HashMap per thread: 17" locally
*
* @author Anthony Goubard - Japplis
*/
@ -83,14 +85,16 @@ public class CalculateAverage_japplis {
int startIndex = handleSplitLine(buffer, readCount);
Runnable countAverageRun = () -> {
int bufferIndex = startIndex;
Map<String, IntSummaryStatistics> blockCityMeasurementMap = new HashMap<>();
try {
while (bufferIndex < readCount) {
bufferIndex = readNextLine(bufferIndex, buffer);
bufferIndex = readNextLine(bufferIndex, buffer, blockCityMeasurementMap);
}
}
catch (ArrayIndexOutOfBoundsException ex) {
// Done reading and parsing the buffer
}
mergeBlockResults(blockCityMeasurementMap);
readFileLock.release();
};
return countAverageRun;
@ -128,7 +132,7 @@ public class CalculateAverage_japplis {
for (int i = 0; i < splitLineBytes.length; i++) {
splitLineBytes[i] = previousBlockLastLine.get(i);
}
readNextLine(0, splitLineBytes);
readNextLine(0, splitLineBytes, cityMeasurementMap);
return bufferIndex;
}
@ -154,7 +158,7 @@ public class CalculateAverage_japplis {
return startIndex;
}
private int readNextLine(int bufferIndex, byte[] buffer) {
private int readNextLine(int bufferIndex, byte[] buffer, Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
int startLineIndex = bufferIndex;
while (buffer[bufferIndex] != ';')
bufferIndex++;
@ -166,7 +170,7 @@ public class CalculateAverage_japplis {
bufferIndex++;
if (temperature <= -precisionLimitTenth || temperature >= precisionLimitTenth)
bufferIndex++;
addTemperature(city, temperature);
addTemperature(city, temperature, blockCityMeasurementMap);
return bufferIndex;
}
@ -187,12 +191,20 @@ public class CalculateAverage_japplis {
return temperature;
}
private void addTemperature(String city, int temperature) {
cityMeasurementMap.compute(city, (town, measurement) -> {
if (measurement == null)
private void addTemperature(String city, int temperature, Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
IntSummaryStatistics measurement = blockCityMeasurementMap.get(city);
if (measurement == null) {
measurement = new IntSummaryStatistics();
blockCityMeasurementMap.put(city, measurement);
}
measurement.accept(temperature);
return measurement;
}
private void mergeBlockResults(Map<String, IntSummaryStatistics> blockCityMeasurementMap) {
blockCityMeasurementMap.forEach((city, measurement) -> {
IntSummaryStatistics oldMeasurement = cityMeasurementMap.putIfAbsent(city, measurement);
if (oldMeasurement != null)
oldMeasurement.combine(measurement);
});
}