Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Lazy strings init, custom station map (#336)

Browse Source
This commit is contained in:
Roman Romanchuk 2024-01-12 09:52:32 +01:00 committed by GitHub
parent 10f425e2b6
commit 594f6c4e5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 36 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/main/java/dev/morling/onebrc/CalculateAverage_fatroom.java
View File

@ -69,7 +69,7 @@ public class CalculateAverage_fatroom {
long fileSize = file.length(); long fileSize = file.length();
long position = 0; long position = 0;
List<Callable<Map<Station, MeasurementAggregator>>> tasks = new LinkedList<>(); List<Callable<StationMap>> tasks = new LinkedList<>();
while (position < fileSize) { while (position < fileSize) {
long end = Math.min(position + SEGMENT_LENGTH, fileSize); long end = Math.min(position + SEGMENT_LENGTH, fileSize);
int length = (int) (end - position); int length = (int) (end - position);
@ -88,7 +88,7 @@ public class CalculateAverage_fatroom {
for (var future : executor.invokeAll(tasks)) { for (var future : executor.invokeAll(tasks)) {
var segmentAggregates = future.get(); var segmentAggregates = future.get();
for (var entry : segmentAggregates.entrySet()) { for (var entry : segmentAggregates.entrySet()) {
aggregates.merge(entry.getKey().toString(), entry.getValue(), MeasurementAggregator::combineWith); aggregates.merge(entry.getKey(), entry.value, MeasurementAggregator::combineWith);
} }
} }
executor.shutdown(); executor.shutdown();
@ -99,10 +99,8 @@ public class CalculateAverage_fatroom {
System.out.println(aggregates); System.out.println(aggregates);
} }
private static Map<Station, MeasurementAggregator> processBuffer(MappedByteBuffer source, int length) { private static StationMap processBuffer(MappedByteBuffer source, int length) {
StationMap aggregates = new StationMap();
Map<Station, MeasurementAggregator> aggregates = new HashMap<>(500);
Station station;
byte[] buffer = new byte[200]; byte[] buffer = new byte[200];
byte[] measurement = new byte[5]; byte[] measurement = new byte[5];
int measurementLength; int measurementLength;
@ -113,7 +111,6 @@ public class CalculateAverage_fatroom {
hash = 31 * hash + b; hash = 31 * hash + b;
buffer[idx++] = b; buffer[idx++] = b;
if (b == ';') { if (b == ';') {
station = new Station(hash, buffer, idx - 1);
measurementLength = 3; measurementLength = 3;
measurement[0] = source.get(++i); measurement[0] = source.get(++i);
measurement[1] = source.get(++i); measurement[1] = source.get(++i);
@ -127,7 +124,7 @@ public class CalculateAverage_fatroom {
measurementLength++; measurementLength++;
} }
} }
aggregates.computeIfAbsent(station, s -> new MeasurementAggregator()).consume(parseMeasurement(measurement, measurementLength)); aggregates.get(hash, buffer, idx - 1).consume(parseMeasurement(measurement, measurementLength));
idx = 0; idx = 0;
hash = 1; hash = 1;
} }
@ -148,28 +145,47 @@ public class CalculateAverage_fatroom {
static class Station { static class Station {
private byte[] bytes; private byte[] bytes;
private int hash; private int hash;
private MeasurementAggregator value;
private Station next;
public Station(int hash, byte[] bytes, int length) { public Station(int hash, byte[] bytes, int length, Station next) {
this.hash = hash;
this.bytes = new byte[length]; this.bytes = new byte[length];
System.arraycopy(bytes, 0, this.bytes, 0, length); System.arraycopy(bytes, 0, this.bytes, 0, length);
this.hash = hash; this.value = new MeasurementAggregator();
this.next = next;
} }
public String toString() { public String getKey() {
return new String(bytes, 0, bytes.length, StandardCharsets.UTF_8); return new String(bytes, 0, bytes.length, StandardCharsets.UTF_8);
} }
}
@Override static class StationMap {
public boolean equals(Object o) { private Station[] stations = new Station[16384];
Station station = (Station) o;
if (hash != station.hash) MeasurementAggregator get(int hash, byte[] buffer, int length) {
return false; int bucketId = hash & 0x3fff;
return Arrays.equals(bytes, station.bytes); Station entry = stations[bucketId];
while (entry != null) {
if (entry.hash == hash && Arrays.equals(entry.bytes, 0, entry.bytes.length, buffer, 0, length)) {
return entry.value;
}
entry = entry.next;
}
stations[bucketId] = new Station(hash, buffer, length, stations[bucketId]);
return stations[bucketId].value;
} }
@Override private List<Station> entrySet() {
public int hashCode() { List<Station> result = new LinkedList<>();
return hash; for (var station : stations) {
while (station != null) {
result.add(station);
station = station.next;
}
}
return result;
} }
} }
} }