diff --git a/README.md b/README.md index a2276f6..489f305 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ These are the results from running all entries into the challenge on eight cores | # | Result (m:s.ms) | Implementation | JDK | Submitter | Notes | |---|-----------------|--------------------|-----|---------------|-----------| | 1 | 00:02.575 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_merykittyunsafe.java)| 21.0.1-open | [Quan Anh Mai](https://github.com/merykitty) | Quan Anh Mai's implementation, using `Unsafe` | -| 2 | 00:02.904 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_royvanrijn.java)| 21.0.1-graal | [Roy van Rijn](https://github.com/royvanrijn) | GraalVM native binary | -| 3 | 00:03.044 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java)| 21.0.1-graal | [Thomas Wuerthinger](https://github.com/thomaswue), [Quan Anh Mai](https://github.com/merykitty), [Alfonso² Peterssen](https://github.com/mukel) | GraalVM native binary | +| 2 | 00:02.708 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thomaswue.java)| 21.0.1-graal | [Thomas Wuerthinger](https://github.com/thomaswue), [Quan Anh Mai](https://github.com/merykitty), [Alfonso² Peterssen](https://github.com/mukel) | GraalVM native binary | +| 3 | 00:02.855 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_royvanrijn.java)| 21.0.1-graal | [Roy van Rijn](https://github.com/royvanrijn) | GraalVM native binary | | | 00:03.258 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_merykitty.java)| 21.0.1-open | [Quan Anh Mai](https://github.com/merykitty) | | | | 00:03.321 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_artsiomkorzun.java)| 21.0.1-graal | [Artsiom Korzun](https://github.com/artsiomkorzun) | | | | 00:03.539 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_mtopolnik.java)| 21.0.1-graal | [Marko Topolnik](https://github.com/mtopolnik) | | @@ -51,9 +51,11 @@ These are the results from running all entries into the challenge on eight cores | | 00:04.823 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_JamalMulla.java)| 21.0.1-graal | [Jamal Mulla](https://github.com/JamalMulla) | | | | 00:04.959 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_yavuztas.java)| 21.0.1-graal | [Yavuz Tas](https://github.com/yavuztas) | | | | 00:04.996 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ebarlas.java)| 21.0.1-graal | [Elliot Barlas](https://github.com/ebarlas) | | +| | 00:05.218 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_iziamos.java)| 21.0.1-open | [John Ziamos](https://github.com/iziamos) | | | | 00:05.478 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_obourgain.java)| 21.0.1-open | [Olivier Bourgain](https://github.com/obourgain) | | | | 00:05.979 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_spullara.java)| 21.0.1-graal | [Sam Pullara](https://github.com/spullara) | | | | 00:06.166 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_isolgpus.java)| 21.0.1-open | [Jamie Stansfield](https://github.com/isolgpus) | | +| | 00:06.576 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_asun.java)| 21.0.1-open | [Alan Sun](https://github.com/asun) | | | | 00:06.715 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_algirdasrascius.java)| 21.0.1-open | [Algirdas Raščius](https://github.com/algirdasrascius) | | | | 00:07.730 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_jotschi.java)| 21.0.1-open | [Johannes Schüth](https://github.com/jotschi) | | | | 00:07.843 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_zerninv.java)| 21.0.1-open | [zerninv](https://github.com/zerninv) | | @@ -65,25 +67,25 @@ These are the results from running all entries into the challenge on eight cores | | 00:08.517 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ags313.java)| 21.0.1-graal | [ags](https://github.com/ags313) | | | | 00:08.892 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_fatroom.java)| 21.0.1-open | [Roman Romanchuk](https://github.com/fatroom) | | | | 00:09.020 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_yemreinci.java)| 21.0.1-open | [yemreinci](https://github.com/yemreinci) | | -| | 00:09.046 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_iziamos.java)| 21.0.1-open | [John Ziamos](https://github.com/iziamos) | | | | 00:09.071 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_gabrielreid.java)| 21.0.1-open | [Gabriel Reid](https://github.com/gabrielreid) | | | | 00:09.117 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_kuduwa-keshavram.java)| 21.0.1-graal | [Keshavram Kuduwa](https://github.com/kuduwa-keshavram) | | | | 00:09.352 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_filiphr.java)| 21.0.1-graal | [Filip Hrisafov](https://github.com/filiphr) | | | | 00:09.867 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ricardopieper.java)| 21.0.1-graal | [Ricardo Pieper](https://github.com/ricardopieper) | | | | 00:10.127 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_artpar.java)| 21.0.1-open | [Parth Mudgal](https://github.com/artpar) | | +| | 00:10.553 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_C5H12O5.java)| 21.0.1-graal | [Xylitol](https://github.com/C5H12O5) | | | | 00:10.949 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_roman-r-m.java)| 21.0.1-graal | [Roman Musin](https://github.com/roman-r-m) | | | | 00:10.473 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_raipc.java)| 21.0.1-open | [Anton Rybochkin](https://github.com/raipc) | | -| | 00:10.642 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_asun.java)| 21.0.1-open | [Alan Sun](https://github.com/asun) | | | | 00:11.119 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_lawrey.java)| 21.0.1-open | [lawrey](https://github.com/lawrey) | | | | 00:11.167 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_palmr.java)| 21.0.1-open | [Nick Palmer](https://github.com/palmr) | | +| | 00:11.405 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_imrafaelmerino.java)| 21.0.1-graal | [Rafael Merino García](https://github.com/imrafaelmerino) | | | | 00:11.805 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_coolmineman.java)| 21.0.1-graal | [Cool_Mineman](https://github.com/coolmineman) | | | | 00:11.934 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_arjenvaneerde.java)| 21.0.1-open | [arjenvaneerde](https://github.com/arjenvaneerde) | | | | 00:11.987 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_flippingbits.java)| 21.0.1-graal | [Stefan Sprenger](https://github.com/flippingbits) | | | | 00:12.220 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_richardstartin.java)| 21.0.1-open | [Richard Startin](https://github.com/richardstartin) | | -| | 00:12.471 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_SamuelYvon.java)| 21.0.1-graal | [Samuel Yvon](https://github.com/SamuelYvon) | | -| | 00:12.557 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_imrafaelmerino.java)| 21.0.1-graal | [Rafael Merino García](https://github.com/imrafaelmerino) | | +| | 00:12.495 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_SamuelYvon.java)| 21.0.1-graal | [Samuel Yvon](https://github.com/SamuelYvon) | GraalVM native binary | | | 00:12.565 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_japplis.java)| 21.0.1-open | [Anthony Goubard](https://github.com/japplis) | | | | 00:12.568 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_MeanderingProgrammer.java)| 21.0.1-graal | [Vlad](https://github.com/MeanderingProgrammer) | | +| | 00:13.013 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_thanhtrinity.java)| 21.0.1-graal | [Thanh Duong](https://github.com/thanhtrinity) | | | | 00:13.148 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_charlibot.java)| 21.0.1-open | [Charlie Evans](https://github.com/charlibot) | | | | 00:13.623 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_phd3.java)| 21.0.1-open | [Pratham](https://github.com/phd3) | | | | 00:13.817 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_entangled90.java)| 21.0.1-open | [Carlo](https://github.com/entangled90) | | @@ -91,13 +93,13 @@ These are the results from running all entries into the challenge on eight cores | | 00:14.867 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_berry120.java)| 21.0.1-open | [Michael Berry](https://github.com/berry120) | | | | 00:15.662 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_semotpan.java)| 21.0.1-open | [Serghei Motpan](https://github.com/semotpan) | | | | 00:15.752 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_jbachorik.java)| 21.0.1-tem | [Jaroslav Bachorik](https://github.com/jbachorik) | | +| | 00:16.379 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_ianopolous.java)| 21.0.1-open | [Dr Ian Preston](https://github.com/ianopolous) | | | | 00:17.490 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_kgeri.java)| 21.0.1-open | [Gergely Kiss](https://github.com/kgeri) | | | | 00:17.815 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_hallvard.java)| 21.0.1-open | [Hallvard Trætteberg](https://github.com/hallvard) | | | | 00:18.251 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_seijikun.java)| 21.0.1-graal | [Markus Ebner](https://github.com/seijikun) | | | | 00:18.448 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_moysesb.java)| 21.0.1-open | [Moysés Borges Furtado](https://github.com/moysesb) | | | | 00:18.771 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_davecom.java)| 21.0.1-graal | [David Kopec](https://github.com/davecom) | | | | 00:19.357 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_truelive.java)| 21.0.1-graalce | [Roman Schweitzer](https://github.com/truelive) | | -| | 00:20.095 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_C5H12O5.java)| 21.0.1-graal | [Xylitol](https://github.com/C5H12O5) | | | | 00:21.989 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_couragelee.java)| 21.0.1-open | [couragelee](https://github.com/couragelee) | | | | 00:22.457 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_rby.java)| 21.0.1-open | [Ramzi Ben Yahya](https://github.com/rby) | | | | 00:24.528 | [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_gnabyl.java)| 21.0.1-open | [Bang NGUYEN](https://github.com/gnabyl) | | diff --git a/calculate_average_ianopolous.sh b/calculate_average_ianopolous.sh old mode 100644 new mode 100755 diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java b/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java index 8d261ef..9d0027c 100644 --- a/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java +++ b/src/main/java/dev/morling/onebrc/CalculateAverage_abeobk.java @@ -14,217 +14,217 @@ * limitations under the License. */ - package dev.morling.onebrc; +package dev.morling.onebrc; - import java.io.IOException; - import java.lang.foreign.Arena; - import java.lang.reflect.Field; - import java.nio.channels.FileChannel; - import java.nio.channels.FileChannel.MapMode; - import java.nio.charset.StandardCharsets; - import java.nio.file.Path; - import java.nio.file.StandardOpenOption; - import java.util.TreeMap; - import sun.misc.Unsafe; - - public class CalculateAverage_abeobk { - private static final String FILE = "./measurements.txt"; - private static final int BUCKET_SIZE = 1 << 16; - private static final int BUCKET_MASK = BUCKET_SIZE - 1; - private static final int MAX_STR_LEN = 100; - private static final Unsafe UNSAFE = initUnsafe(); - private static final long[] HASH_MASKS = new long[]{ - 0x0L, - 0xffL, - 0xffffL, - 0xffffffL, - 0xffffffffL, - 0xffffffffffL, - 0xffffffffffffL, - 0xffffffffffffffL, - 0xffffffffffffffffL, }; - - private static Unsafe initUnsafe() { - try { - Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); - theUnsafe.setAccessible(true); - return (Unsafe) theUnsafe.get(Unsafe.class); - } - catch (Exception ex) { - throw new RuntimeException(); - } - } - - // stat - private static class Stat { - private int min; - private int max; - private long sum; - private int count; - - Stat(int v) { - sum = min = max = v; - count = 1; - } - - void add(int val) { - min = Math.min(val, min); - max = Math.max(val, max); - sum += val; - count++; - } - - void merge(Stat other) { - min = Math.min(other.min, min); - max = Math.max(other.max, max); - sum += other.sum; - count += other.count; - } - - public String toString() { - return String.format("%.1f/%.1f/%.1f", min * 0.1, sum * 0.1 / count, max * 0.1); - } - } - - static class Node { - long addr; - int keylen; - int hash; - long[] buf = new long[13]; - Stat stat; - - String key() { - byte[] buf = new byte[MAX_STR_LEN]; - UNSAFE.copyMemory(null, addr, buf, Unsafe.ARRAY_BYTE_BASE_OFFSET, keylen); - return new String(buf, 0, keylen, StandardCharsets.UTF_8); - } - - Node(long a, int kl, int h, int v, long[] b) { - stat = new Stat(v); - addr = a; - keylen = kl; - hash = h; - System.arraycopy(b, 0, buf, 0, Math.ceilDiv(kl, 8)); - } - - boolean contentEquals(final long[] other_buf) { - int k = keylen / 8; - int r = keylen % 8; - // Since the city name is most likely shorter than 16 characters - // this should be faster than typical conditional checks - long sum = 0; - for (int i = 0; i < k; i++) { - sum += buf[i] ^ other_buf[i]; - } - sum += (buf[k] ^ other_buf[k]) & HASH_MASKS[r]; - return sum == 0; - } - } - - // split into chunks - static long[] slice(long start_addr, long end_addr, long chunk_size, int cpu_cnt) { - long[] ptrs = new long[cpu_cnt + 1]; - ptrs[0] = start_addr; - for (int i = 1; i < cpu_cnt; i++) { - long addr = start_addr + i * chunk_size; - while (addr < end_addr && UNSAFE.getByte(addr++) != '\n') - ; - ptrs[i] = Math.min(addr, end_addr); - } - ptrs[cpu_cnt] = end_addr; - return ptrs; - } - - public static void main(String[] args) throws InterruptedException, IOException { - int cpu_cnt = Runtime.getRuntime().availableProcessors() / 2; - try (var file = FileChannel.open(Path.of(FILE), StandardOpenOption.READ)) { - long start_addr = file.map(MapMode.READ_ONLY, 0, file.size(), Arena.global()).address(); - long file_size = file.size(); - long end_addr = start_addr + file_size; - long chunk_size = Math.ceilDiv(file_size, cpu_cnt); - - // processing - var threads = new Thread[cpu_cnt]; - var maps = new Node[cpu_cnt][]; - var ptrs = slice(start_addr, end_addr, chunk_size, cpu_cnt); - - for (int i = 0; i < cpu_cnt; i++) { - int thread_id = i; - long start = ptrs[i]; - long end = ptrs[i + 1]; - maps[i] = new Node[BUCKET_SIZE + 16]; // extra space for collisions - - (threads[i] = new Thread(() -> { - long addr = start; - var map = maps[thread_id]; - long[] buf = new long[13]; - // parse loop - while (addr < end) { - int idx = 0; - long hash = 0; - long word = 0; - long row_addr = addr; - int semi_pos = 8; - while (semi_pos == 8) { - word = UNSAFE.getLong(addr); - buf[idx++] = word; - // idea from thomaswue & royvanrijn - long xor_semi = word ^ 0x3b3b3b3b3b3b3b3bL; // xor with ;;;;;;;; - long semipos_code = (xor_semi - 0x0101010101010101L) & ~xor_semi & 0x8080808080808080L; - semi_pos = Long.numberOfTrailingZeros(semipos_code) >>> 3; - addr += semi_pos; - hash ^= word & HASH_MASKS[semi_pos]; - } - - int hash32 = (int) (hash ^ (hash >>> 31)); - int keylen = (int) (addr - row_addr); - - // great idea from merykitty (Quan Anh Mai) - long num_word = UNSAFE.getLong(++addr); - int dot_pos = Long.numberOfTrailingZeros(~num_word & 0x10101000); - addr += (dot_pos >>> 3) + 3; - - int shift = 28 - dot_pos; - long signed = (~num_word << 59) >> 63; - long dsmask = ~(signed & 0xFF); - long digits = ((num_word & dsmask) << shift) & 0x0F000F0F00L; - long abs_val = ((digits * 0x640a0001) >>> 32) & 0x3FF; - int val = (int) ((abs_val ^ signed) - signed); - - int bucket = (hash32 & BUCKET_MASK); - while (true) { - var node = map[bucket]; - if (node == null) { - map[bucket] = new Node(row_addr, keylen, hash32, val, buf); - break; - } - if (node.keylen == keylen && node.hash == hash32 && node.contentEquals(buf)) { - node.stat.add(val); - break; - } - bucket++; - } - } - })).start(); - } - - // join all - for (var thread : threads) - thread.join(); - - // collect results - TreeMap ms = new TreeMap<>(); - for (var map : maps) { - for (var node : map) { - if (node == null) - continue; - var stat = ms.putIfAbsent(node.key(), node.stat); - if (stat != null) - stat.merge(node.stat); - } - } - - System.out.println(ms); - } - } - } \ No newline at end of file +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.reflect.Field; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.TreeMap; +import sun.misc.Unsafe; + +public class CalculateAverage_abeobk { + private static final String FILE = "./measurements.txt"; + private static final int BUCKET_SIZE = 1 << 16; + private static final int BUCKET_MASK = BUCKET_SIZE - 1; + private static final int MAX_STR_LEN = 100; + private static final Unsafe UNSAFE = initUnsafe(); + private static final long[] HASH_MASKS = new long[]{ + 0x0L, + 0xffL, + 0xffffL, + 0xffffffL, + 0xffffffffL, + 0xffffffffffL, + 0xffffffffffffL, + 0xffffffffffffffL, + 0xffffffffffffffffL, }; + + private static Unsafe initUnsafe() { + try { + Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); + theUnsafe.setAccessible(true); + return (Unsafe) theUnsafe.get(Unsafe.class); + } + catch (Exception ex) { + throw new RuntimeException(); + } + } + + // stat + private static class Stat { + private int min; + private int max; + private long sum; + private int count; + + Stat(int v) { + sum = min = max = v; + count = 1; + } + + void add(int val) { + min = Math.min(val, min); + max = Math.max(val, max); + sum += val; + count++; + } + + void merge(Stat other) { + min = Math.min(other.min, min); + max = Math.max(other.max, max); + sum += other.sum; + count += other.count; + } + + public String toString() { + return String.format("%.1f/%.1f/%.1f", min * 0.1, sum * 0.1 / count, max * 0.1); + } + } + + static class Node { + long addr; + int keylen; + int hash; + long[] buf = new long[13]; + Stat stat; + + String key() { + byte[] buf = new byte[MAX_STR_LEN]; + UNSAFE.copyMemory(null, addr, buf, Unsafe.ARRAY_BYTE_BASE_OFFSET, keylen); + return new String(buf, 0, keylen, StandardCharsets.UTF_8); + } + + Node(long a, int kl, int h, int v, long[] b) { + stat = new Stat(v); + addr = a; + keylen = kl; + hash = h; + System.arraycopy(b, 0, buf, 0, Math.ceilDiv(kl, 8)); + } + + boolean contentEquals(final long[] other_buf) { + int k = keylen / 8; + int r = keylen % 8; + // Since the city name is most likely shorter than 16 characters + // this should be faster than typical conditional checks + long sum = 0; + for (int i = 0; i < k; i++) { + sum += buf[i] ^ other_buf[i]; + } + sum += (buf[k] ^ other_buf[k]) & HASH_MASKS[r]; + return sum == 0; + } + } + + // split into chunks + static long[] slice(long start_addr, long end_addr, long chunk_size, int cpu_cnt) { + long[] ptrs = new long[cpu_cnt + 1]; + ptrs[0] = start_addr; + for (int i = 1; i < cpu_cnt; i++) { + long addr = start_addr + i * chunk_size; + while (addr < end_addr && UNSAFE.getByte(addr++) != '\n') + ; + ptrs[i] = Math.min(addr, end_addr); + } + ptrs[cpu_cnt] = end_addr; + return ptrs; + } + + public static void main(String[] args) throws InterruptedException, IOException { + int cpu_cnt = Runtime.getRuntime().availableProcessors() / 2; + try (var file = FileChannel.open(Path.of(FILE), StandardOpenOption.READ)) { + long start_addr = file.map(MapMode.READ_ONLY, 0, file.size(), Arena.global()).address(); + long file_size = file.size(); + long end_addr = start_addr + file_size; + long chunk_size = Math.ceilDiv(file_size, cpu_cnt); + + // processing + var threads = new Thread[cpu_cnt]; + var maps = new Node[cpu_cnt][]; + var ptrs = slice(start_addr, end_addr, chunk_size, cpu_cnt); + + for (int i = 0; i < cpu_cnt; i++) { + int thread_id = i; + long start = ptrs[i]; + long end = ptrs[i + 1]; + maps[i] = new Node[BUCKET_SIZE + 16]; // extra space for collisions + + (threads[i] = new Thread(() -> { + long addr = start; + var map = maps[thread_id]; + long[] buf = new long[13]; + // parse loop + while (addr < end) { + int idx = 0; + long hash = 0; + long word = 0; + long row_addr = addr; + int semi_pos = 8; + while (semi_pos == 8) { + word = UNSAFE.getLong(addr); + buf[idx++] = word; + // idea from thomaswue & royvanrijn + long xor_semi = word ^ 0x3b3b3b3b3b3b3b3bL; // xor with ;;;;;;;; + long semipos_code = (xor_semi - 0x0101010101010101L) & ~xor_semi & 0x8080808080808080L; + semi_pos = Long.numberOfTrailingZeros(semipos_code) >>> 3; + addr += semi_pos; + hash ^= word & HASH_MASKS[semi_pos]; + } + + int hash32 = (int) (hash ^ (hash >>> 31)); + int keylen = (int) (addr - row_addr); + + // great idea from merykitty (Quan Anh Mai) + long num_word = UNSAFE.getLong(++addr); + int dot_pos = Long.numberOfTrailingZeros(~num_word & 0x10101000); + addr += (dot_pos >>> 3) + 3; + + int shift = 28 - dot_pos; + long signed = (~num_word << 59) >> 63; + long dsmask = ~(signed & 0xFF); + long digits = ((num_word & dsmask) << shift) & 0x0F000F0F00L; + long abs_val = ((digits * 0x640a0001) >>> 32) & 0x3FF; + int val = (int) ((abs_val ^ signed) - signed); + + int bucket = (hash32 & BUCKET_MASK); + while (true) { + var node = map[bucket]; + if (node == null) { + map[bucket] = new Node(row_addr, keylen, hash32, val, buf); + break; + } + if (node.keylen == keylen && node.hash == hash32 && node.contentEquals(buf)) { + node.stat.add(val); + break; + } + bucket++; + } + } + })).start(); + } + + // join all + for (var thread : threads) + thread.join(); + + // collect results + TreeMap ms = new TreeMap<>(); + for (var map : maps) { + for (var node : map) { + if (node == null) + continue; + var stat = ms.putIfAbsent(node.key(), node.stat); + if (stat != null) + stat.merge(node.stat); + } + } + + System.out.println(ms); + } + } +} \ No newline at end of file