Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

copy city byte array only when creating a new record (#653)

Browse Source
This commit is contained in:
Andrzej Nestoruk 2024-01-31 17:26:22 +01:00 committed by GitHub
parent f55317973c
commit 5728ca9482
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/main/java/dev/morling/onebrc/CalculateAverage_anestoruk.java
View File

@ -49,7 +49,7 @@ public class CalculateAverage_anestoruk {
try (FileChannel channel = FileChannel.open(Path.of(path))) { try (FileChannel channel = FileChannel.open(Path.of(path))) {
final long fileSize = channel.size(); final long fileSize = channel.size();
final long chunkSize = fileSize > 10_000 ? min(Integer.MAX_VALUE - 256, fileSize / cpus) : fileSize; final long chunkSize = calculateChunkSize(fileSize);
final int chunks = (int) ceil((double) fileSize / chunkSize); final int chunks = (int) ceil((double) fileSize / chunkSize);
segment = channel.map(READ_ONLY, 0, fileSize, Arena.global()); segment = channel.map(READ_ONLY, 0, fileSize, Arena.global());
long startOffset = 0; long startOffset = 0;
@ -85,6 +85,18 @@ public class CalculateAverage_anestoruk {
System.out.println(result); System.out.println(result);
} }
private static long calculateChunkSize(long fileSize) {
int divisor = cpus;
long chunkSize;
if (fileSize > 10_000) {
while ((chunkSize = fileSize / divisor) > Integer.MAX_VALUE - 512) {
divisor *= 2;
}
return chunkSize;
}
return fileSize;
}
private static Record[] process(SegmentRange range, MemorySegment segment) { private static Record[] process(SegmentRange range, MemorySegment segment) {
Record[] records = new Record[1024 * 100]; Record[] records = new Record[1024 * 100];
byte[] cityBuffer = new byte[100]; byte[] cityBuffer = new byte[100];
@ -113,23 +125,23 @@ public class CalculateAverage_anestoruk {
} }
} }
int temperature = negative ? -value : value; int temperature = negative ? -value : value;
byte[] city = new byte[cityLength]; addRecord(records, hash, cityBuffer, cityLength, temperature);
System.arraycopy(cityBuffer, 0, city, 0, cityLength);
addResult(records, hash, city, temperature);
} }
return records; return records;
} }
private static void addResult(Record[] records, int hash, byte[] city, int temperature) { private static void addRecord(Record[] records, int hash, byte[] cityBuffer, int cityLength, int temperature) {
int idx = hash % records.length; int idx = hash % records.length;
Record record; Record record;
while ((record = records[idx]) != null) { while ((record = records[idx]) != null) {
if (record.hash == hash && Arrays.equals(record.city, city)) { if (record.hash == hash && Arrays.equals(record.city, 0, record.city.length, cityBuffer, 0, cityLength)) {
record.add(temperature); record.add(temperature);
return; return;
} }
idx = (idx + 1) % records.length; idx = (idx + 1) % records.length;
} }
byte[] city = new byte[cityLength];
System.arraycopy(cityBuffer, 0, city, 0, cityLength);
records[idx] = new Record(hash, city, temperature); records[idx] = new Record(hash, city, temperature);
} }