Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use a hash key for the city as a key in the map

Browse Source
This commit is contained in:
Filip Hrisafov 2024-01-03 22:34:23 +01:00 committed by Gunnar Morling
parent 4a483b4097
commit f5f3a41045
1 changed files with 40 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/dev/morling/onebrc/CalculateAverage_filiphr.java
View File

@ -40,6 +40,7 @@ import java.util.stream.StreamSupport;
* Adding memory mapped files: 0m 55s (based on bjhara's submission)
* Using big decimal and iterating the buffer once: 0m 20s
* Using long parse: 0m 11s
* Using array hash code for city key: 0m 7.1s
* <p>
* Using 21.0.1 Temurin with ShenandoahGC on Macbook (Intel) Pro
* `sdk use java 21.0.1-tem`
@ -59,11 +60,16 @@ public class CalculateAverage_filiphr {
private static final class Measurement {
private final String city;
private long min = Long.MAX_VALUE;
private long max = Long.MIN_VALUE;
private long sum = 0L;
private long count = 0L;
private Measurement(String city) {
this.city = city;
}
private void add(long value) {
this.min = Math.min(this.min, value);
this.max = Math.max(this.max, value);
@ -72,7 +78,7 @@ public class CalculateAverage_filiphr {
}
public static Measurement combine(Measurement m1, Measurement m2) {
Measurement measurement = new Measurement();
Measurement measurement = new Measurement(m1.city);
measurement.min = Math.min(m1.min, m2.min);
measurement.max = Math.max(m1.max, m2.max);
measurement.sum = m1.sum + m2.sum;
@ -93,7 +99,7 @@ public class CalculateAverage_filiphr {
public static void main(String[] args) throws IOException {
// long start = System.nanoTime();
Map<String, Measurement> measurements;
Map<Integer, Measurement> measurements;
try (FileChannel fileChannel = FileChannel.open(Paths.get(FILE), StandardOpenOption.READ)) {
measurements = fineChannelStream(fileChannel)
.parallel()
@ -101,20 +107,25 @@ public class CalculateAverage_filiphr {
.reduce(Collections.emptyMap(), CalculateAverage_filiphr::mergeMaps);
}
System.out.println(new TreeMap<>(measurements));
Map<String, Measurement> finalMeasurements = new TreeMap<>();
for (Measurement measurement : measurements.values()) {
finalMeasurements.put(measurement.city, measurement);
}
System.out.println(finalMeasurements);
// System.out.println("Done in " + (System.nanoTime() - start) / 1000000 + " ms");
}
private static Map<String, Measurement> mergeMaps(Map<String, Measurement> map1, Map<String, Measurement> map2) {
private static Map<Integer, Measurement> mergeMaps(Map<Integer, Measurement> map1, Map<Integer, Measurement> map2) {
if (map1.isEmpty()) {
return map2;
}
else {
Set<String> cities = new HashSet<>(map1.keySet());
Set<Integer> cities = new HashSet<>(map1.keySet());
cities.addAll(map2.keySet());
Map<String, Measurement> result = HashMap.newHashMap(cities.size());
Map<Integer, Measurement> result = HashMap.newHashMap(cities.size());
for (String city : cities) {
for (Integer city : cities) {
Measurement m1 = map1.get(city);
Measurement m2 = map2.get(city);
if (m2 == null) {
@ -137,34 +148,36 @@ public class CalculateAverage_filiphr {
}
/**
* This is an adapted implementation of the bjhara parseBuffer
* This is an adapted implementation of the bjhara parseBuffer.
* We are using {@code Map<Integer, Measurement>} because creating the string key on every single line is obsolete.
* Instead, we create a hash key from the string, and we use that as a key in the map.
*/
private static Map<String, Measurement> parseBuffer(ByteBuffer bb) {
Map<String, Measurement> measurements = HashMap.newHashMap(415);
private static Map<Integer, Measurement> parseBuffer(ByteBuffer bb) {
Map<Integer, Measurement> measurements = HashMap.newHashMap(415);
int limit = bb.limit();
byte[] buffer = new byte[128];
byte[] cityBuffer = new byte[128];
char[] charArray = new char[8];
CharBuffer charBuffer = CharBuffer.wrap(charArray);
charBuffer.clear();
charBuffer.position(0);
while (bb.position() < limit) {
int bufferIndex = 0;
int cityBufferIndex = 0;
// Iterate through the byte buffer and fill the buffer until we find the separator (;)
// While iterating we are also going to compute the city hash key
int cityKey = 1;
while (bb.position() < limit) {
byte positionByte = bb.get();
if (positionByte == ';') {
break;
}
buffer[bufferIndex++] = positionByte;
cityBuffer[cityBufferIndex++] = positionByte;
cityKey = 31 * cityKey + positionByte;
}
// Create the city
String city = new String(buffer, 0, bufferIndex);
byte lastPositionByte = '\n';
bufferIndex = 0;
int temperatureBufferIndex = 0;
while (bb.position() < limit) {
byte positionByte = bb.get();
if (positionByte == '\r' || positionByte == '\n') {
@ -172,15 +185,20 @@ public class CalculateAverage_filiphr {
break;
}
else if (positionByte != '.') {
charArray[bufferIndex++] = (char) positionByte;
charArray[temperatureBufferIndex++] = (char) positionByte;
}
}
// Create the temperature string
long value = Long.parseLong(charBuffer, 0, bufferIndex, 10);
// Create the temperature
long value = Long.parseLong(charBuffer, 0, temperatureBufferIndex, 10);
measurements.computeIfAbsent(city, k -> new Measurement())
.add(value);
Measurement measurement = measurements.get(cityKey);
if (measurement == null) {
String city = new String(cityBuffer, 0, cityBufferIndex);
measurement = new Measurement(city);
measurements.put(cityKey, measurement);
}
measurement.add(value);
// and get rid of the new line (handle both kinds)
if (lastPositionByte == '\r') {