Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use a hash key for the city as a key in the map

Browse Source
This commit is contained in:
Filip Hrisafov 2024-01-03 22:34:23 +01:00 committed by Gunnar Morling
parent 4a483b4097
commit f5f3a41045
1 changed files with 40 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/dev/morling/onebrc/CalculateAverage_filiphr.java
View File

@ -40,6 +40,7 @@ import java.util.stream.StreamSupport;
* Adding memory mapped files: 0m 55s (based on bjhara's submission) * Adding memory mapped files: 0m 55s (based on bjhara's submission)
* Using big decimal and iterating the buffer once: 0m 20s * Using big decimal and iterating the buffer once: 0m 20s
* Using long parse: 0m 11s * Using long parse: 0m 11s
* Using array hash code for city key: 0m 7.1s
* <p> * <p>
* Using 21.0.1 Temurin with ShenandoahGC on Macbook (Intel) Pro * Using 21.0.1 Temurin with ShenandoahGC on Macbook (Intel) Pro
* `sdk use java 21.0.1-tem` * `sdk use java 21.0.1-tem`
@ -59,11 +60,16 @@ public class CalculateAverage_filiphr {
private static final class Measurement { private static final class Measurement {
private final String city;
private long min = Long.MAX_VALUE; private long min = Long.MAX_VALUE;
private long max = Long.MIN_VALUE; private long max = Long.MIN_VALUE;
private long sum = 0L; private long sum = 0L;
private long count = 0L; private long count = 0L;
private Measurement(String city) {
this.city = city;
}
private void add(long value) { private void add(long value) {
this.min = Math.min(this.min, value); this.min = Math.min(this.min, value);
this.max = Math.max(this.max, value); this.max = Math.max(this.max, value);
@ -72,7 +78,7 @@ public class CalculateAverage_filiphr {
} }
public static Measurement combine(Measurement m1, Measurement m2) { public static Measurement combine(Measurement m1, Measurement m2) {
Measurement measurement = new Measurement(); Measurement measurement = new Measurement(m1.city);
measurement.min = Math.min(m1.min, m2.min); measurement.min = Math.min(m1.min, m2.min);
measurement.max = Math.max(m1.max, m2.max); measurement.max = Math.max(m1.max, m2.max);
measurement.sum = m1.sum + m2.sum; measurement.sum = m1.sum + m2.sum;
@ -93,7 +99,7 @@ public class CalculateAverage_filiphr {
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {
// long start = System.nanoTime(); // long start = System.nanoTime();
Map<String, Measurement> measurements; Map<Integer, Measurement> measurements;
try (FileChannel fileChannel = FileChannel.open(Paths.get(FILE), StandardOpenOption.READ)) { try (FileChannel fileChannel = FileChannel.open(Paths.get(FILE), StandardOpenOption.READ)) {
measurements = fineChannelStream(fileChannel) measurements = fineChannelStream(fileChannel)
.parallel() .parallel()
@ -101,20 +107,25 @@ public class CalculateAverage_filiphr {
.reduce(Collections.emptyMap(), CalculateAverage_filiphr::mergeMaps); .reduce(Collections.emptyMap(), CalculateAverage_filiphr::mergeMaps);
} }
System.out.println(new TreeMap<>(measurements)); Map<String, Measurement> finalMeasurements = new TreeMap<>();
for (Measurement measurement : measurements.values()) {
finalMeasurements.put(measurement.city, measurement);
}
System.out.println(finalMeasurements);
// System.out.println("Done in " + (System.nanoTime() - start) / 1000000 + " ms"); // System.out.println("Done in " + (System.nanoTime() - start) / 1000000 + " ms");
} }
private static Map<String, Measurement> mergeMaps(Map<String, Measurement> map1, Map<String, Measurement> map2) { private static Map<Integer, Measurement> mergeMaps(Map<Integer, Measurement> map1, Map<Integer, Measurement> map2) {
if (map1.isEmpty()) { if (map1.isEmpty()) {
return map2; return map2;
} }
else { else {
Set<String> cities = new HashSet<>(map1.keySet()); Set<Integer> cities = new HashSet<>(map1.keySet());
cities.addAll(map2.keySet()); cities.addAll(map2.keySet());
Map<String, Measurement> result = HashMap.newHashMap(cities.size()); Map<Integer, Measurement> result = HashMap.newHashMap(cities.size());
for (String city : cities) { for (Integer city : cities) {
Measurement m1 = map1.get(city); Measurement m1 = map1.get(city);
Measurement m2 = map2.get(city); Measurement m2 = map2.get(city);
if (m2 == null) { if (m2 == null) {
@ -137,34 +148,36 @@ public class CalculateAverage_filiphr {
} }
/** /**
* This is an adapted implementation of the bjhara parseBuffer * This is an adapted implementation of the bjhara parseBuffer.
* We are using {@code Map<Integer, Measurement>} because creating the string key on every single line is obsolete.
* Instead, we create a hash key from the string, and we use that as a key in the map.
*/ */
private static Map<String, Measurement> parseBuffer(ByteBuffer bb) { private static Map<Integer, Measurement> parseBuffer(ByteBuffer bb) {
Map<String, Measurement> measurements = HashMap.newHashMap(415); Map<Integer, Measurement> measurements = HashMap.newHashMap(415);
int limit = bb.limit(); int limit = bb.limit();
byte[] buffer = new byte[128]; byte[] cityBuffer = new byte[128];
char[] charArray = new char[8]; char[] charArray = new char[8];
CharBuffer charBuffer = CharBuffer.wrap(charArray); CharBuffer charBuffer = CharBuffer.wrap(charArray);
charBuffer.clear(); charBuffer.clear();
charBuffer.position(0); charBuffer.position(0);
while (bb.position() < limit) { while (bb.position() < limit) {
int bufferIndex = 0; int cityBufferIndex = 0;
// Iterate through the byte buffer and fill the buffer until we find the separator (;) // Iterate through the byte buffer and fill the buffer until we find the separator (;)
// While iterating we are also going to compute the city hash key
int cityKey = 1;
while (bb.position() < limit) { while (bb.position() < limit) {
byte positionByte = bb.get(); byte positionByte = bb.get();
if (positionByte == ';') { if (positionByte == ';') {
break; break;
} }
buffer[bufferIndex++] = positionByte; cityBuffer[cityBufferIndex++] = positionByte;
cityKey = 31 * cityKey + positionByte;
} }
// Create the city
String city = new String(buffer, 0, bufferIndex);
byte lastPositionByte = '\n'; byte lastPositionByte = '\n';
bufferIndex = 0; int temperatureBufferIndex = 0;
while (bb.position() < limit) { while (bb.position() < limit) {
byte positionByte = bb.get(); byte positionByte = bb.get();
if (positionByte == '\r' || positionByte == '\n') { if (positionByte == '\r' || positionByte == '\n') {
@ -172,15 +185,20 @@ public class CalculateAverage_filiphr {
break; break;
} }
else if (positionByte != '.') { else if (positionByte != '.') {
charArray[bufferIndex++] = (char) positionByte; charArray[temperatureBufferIndex++] = (char) positionByte;
} }
} }
// Create the temperature string // Create the temperature
long value = Long.parseLong(charBuffer, 0, bufferIndex, 10); long value = Long.parseLong(charBuffer, 0, temperatureBufferIndex, 10);
measurements.computeIfAbsent(city, k -> new Measurement()) Measurement measurement = measurements.get(cityKey);
.add(value); if (measurement == null) {
String city = new String(cityBuffer, 0, cityBufferIndex);
measurement = new Measurement(city);
measurements.put(cityKey, measurement);
}
measurement.add(value);
// and get rid of the new line (handle both kinds) // and get rid of the new line (handle both kinds)
if (lastPositionByte == '\r') { if (lastPositionByte == '\r') {