Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

hundredwatt - 2nd submission (#338)

Browse Source 
* check full hash before comparing

* implement merykitty suggestions to simplify temperature masking; required refactoring to little-endian

* standalone script for offline Perfect Hash seed searching

* stop using an oversized hash table

---------

Co-authored-by: Jason Nochlin <hundredwatt@users.noreply.github.com>
This commit is contained in:
Jason Nochlin 2024-01-12 01:03:31 -07:00 committed by GitHub
parent 80c5c2eb3e
commit 6181996678
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 195 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
src/main/java/dev/morling/onebrc/CalculateAverage_hundredwatt.java
View File

@ -17,6 +17,7 @@ package dev.morling.onebrc;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
@ -36,7 +37,7 @@ public class CalculateAverage_hundredwatt {
private static final long FILE_CHUNK_SIZE = CHUNK_SIZE - MAX_ROW_SIZE;
public static final int TEMPERATURE_SLOTS = 5003; // prime number
private static final short[] TEMPERATURES = new short[TEMPERATURE_SLOTS];
private static final long PERFECT_HASH_SEED = -5353381455852817461L;
private static final long PERFECT_HASH_SEED = -1982870890352534081L;
// Construct a perfect hash function mapping temperatures encoded as longs (e.g., 0x2d342e3000000000 for -4.3) to
// the corresponding short integer (e.g., -43).
@ -46,17 +47,17 @@ public class CalculateAverage_hundredwatt {
Map<Long, Short> decodeTemperatureMap = new HashMap<>();
for (short i = -999; i <= 999; i++) {
long word = 0;
int shift = 56;
int shift = 0;
if (i < 0) {
word |= ((long) '-') << shift;
shift -= 8;
shift += 8;
}
if (Math.abs(i) >= 100) {
int hh = Math.abs(i) / 100;
int tt = (Math.abs(i) - hh * 100) / 10;
word |= ((long) (hh + '0')) << shift;
shift -= 8;
shift += 8;
word |= ((long) (tt + '0')) << shift;
}
else {
@ -64,9 +65,9 @@ public class CalculateAverage_hundredwatt {
// convert to ascii
word |= ((long) (tt + '0')) << shift;
}
shift -= 8;
shift += 8;
word |= ((long) '.') << shift;
shift -= 8;
shift += 8;
int uu = Math.abs(i) % 10;
word |= ((long) (uu + '0')) << shift;
@ -74,31 +75,6 @@ public class CalculateAverage_hundredwatt {
decodeTemperatureMap.put(word, i);
}
// Brute force to find seed:
// Random rand = new Random(System.nanoTime());
// int max = 0;
// int attempts = 0;
// while (true) {
// BitSet bs = new BitSet(5003);
// var seed = rand.nextLong();
// seed |= 0b1; // make sure it's odd
// for (var word : decodeTemperatureMap.keySet()) {
// var h = (word * seed) & ~(1L << 63);
// var pos = (int) (h % 5003);
// bs.set(pos);
// }
// ;
// var c = bs.cardinality();
// if (c == decodeTemperatureMap.size()) {
// System.out.println("seed: " + seed + " cardinality: " + c + " max cardinality: " + max);
// break;
// }
// max = Math.max(max, c);
// if (attempts % 100_000 == 0)
// System.out.println("seed: " + seed + " cardinality: " + c + " max cardinality: " + max);
// attempts++;
// }
decodeTemperatureMap.entrySet().stream().forEach(e -> {
var word = e.getKey();
var h = (word * PERFECT_HASH_SEED) & ~(1L << 63);
@ -144,11 +120,12 @@ public class CalculateAverage_hundredwatt {
}
static class HashTable {
private static final int INITIAL_SIZE = 128 * 1024;
private static final int INITIAL_SIZE = 16 * 1024;
private static final float LOAD_FACTOR = 0.75f;
private static final int GROW_FACTOR = 4;
private final long[][] KEYS = new long[INITIAL_SIZE][];
private final Record[] VALUES = new Record[INITIAL_SIZE];
private final long[] HASHES = new long[INITIAL_SIZE];
private int size = INITIAL_SIZE;
public HashTable() {
@ -162,13 +139,14 @@ public class CalculateAverage_hundredwatt {
// linear probing
int i = 0;
while (KEYS[idx] != null && (0 != Arrays.compareUnsigned(KEYS[idx], 0, KEYS[idx].length, key, 0, length))) {
while (KEYS[idx] != null && (HASHES[idx] != hash) && (0 != Arrays.compareUnsigned(KEYS[idx], 0, KEYS[idx].length, key, 0, length))) {
i++;
idx = (idx + 1) & (size - 1);
}
if (KEYS[idx] == null) {
KEYS[idx] = Arrays.copyOf(key, length);
HASHES[idx] = hash;
}
VALUES[idx].updateWith(value);
@ -186,7 +164,7 @@ public class CalculateAverage_hundredwatt {
}
private static String keyToString(long[] key) {
ByteBuffer kb = ByteBuffer.allocate(8 * key.length);
ByteBuffer kb = ByteBuffer.allocate(8 * key.length).order(ByteOrder.LITTLE_ENDIAN);
Arrays.stream(key).forEach(kb::putLong);
// remove trailing '\0' bytes from kb and
@ -194,7 +172,7 @@ public class CalculateAverage_hundredwatt {
byte b;
int limit = kb.position() - 8;
kb.position(limit);
while ((b = kb.get()) != 0 && b != ';') {
while ((b = kb.get()) != 0 && b != ';' && limit < kb.capacity() - 1) {
limit++;
}
@ -202,7 +180,7 @@ public class CalculateAverage_hundredwatt {
byte[] bytes = new byte[limit];
kb.get(bytes);
return new String(bytes).replace("\0", "");
return new String(bytes);
}
private static Record merge(Record v, Record value) {
@ -215,6 +193,7 @@ public class CalculateAverage_hundredwatt {
}
private static int processChunk(ByteBuffer bb, HashTable hashTable, long start, long size) {
bb.order(ByteOrder.LITTLE_ENDIAN);
// Find first entry
while (start != 0 && bb.get() != '\n') {
}
@ -228,6 +207,7 @@ public class CalculateAverage_hundredwatt {
long temperature_hash;
int temperature_pos;
short temperature_value;
int hashInt;
int i = 0;
int end = (int) (size - MAX_ROW_SIZE);
@ -258,29 +238,28 @@ public class CalculateAverage_hundredwatt {
hasvalue = (op1 & op2 & 0x8080808080808080L);
}
hash ^= key[offset]; // unset last word since it will be updated
key[offset] = key[offset] & (-hasvalue); // ';' == 0x3b and -hasvalue is something like 0xff8000..., we can ignore the 0x80 byte since 0x3b & 0x80 == 0
key[offset] = key[offset] & ~(-(hasvalue >> 7));
hash ^= key[offset];
position = position + offset * 8 + (Long.numberOfLeadingZeros(hasvalue)) / 8 + 1; // +1 for \n
position = position + offset * 8 + Long.numberOfTrailingZeros(hasvalue) / 8 + 1; // +1 for \n
// Parse temperature
word = bb.getLong(position);
arg = (word) ^ 0x0101010101010101L * ('\n');
op1 = (arg - 0x0101010101010101L);
op2 = ~(arg);
hasvalue = (op1 & op2 & 0x8080808080808080L);
word = word & ((-hasvalue));
hasvalue = (word - 0x0B0B0B0B0B0B0B0BL) & 0x8080808080808080L;
int newlinePos = Long.numberOfTrailingZeros(hasvalue) - 8;
word = word & (~(-(1L << newlinePos)));
// Perfect hash lookup for temperature
temperature_hash = (word * PERFECT_HASH_SEED) & ~(1L << 63);
temperature_pos = (int) (temperature_hash % TEMPERATURE_SLOTS);
temperature_value = TEMPERATURES[temperature_pos];
position = position + (Long.numberOfLeadingZeros(hasvalue)) / 8 + 1; // +1 for \n
position = position + newlinePos / 8 + 2; // +1 for \n
int hash2 = (int) (hash ^ (hash >> 32));
hashInt = (int) (hash ^ (hash >> 32));
hashTable.putOrMerge(hash2, offset + 1, key, temperature_value);
hashTable.putOrMerge(hashInt, offset + 1, key, temperature_value);
}
return i;
}

170
src/main/java/dev/morling/onebrc/PerfectHashSearch_hundredwatt.java Normal file
View File

@ -0,0 +1,170 @@
/*
* Copyright 2023 The original authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dev.morling.onebrc;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.random.RandomGenerator;
import java.util.random.RandomGeneratorFactory;
/** Offline script used to find the perfect hash seed for CalculateAverage_hundredwatt. */
public class PerfectHashSearch_hundredwatt {
public static final int DESIRED_SLOTS = 5003;
public static final int N_THREADS = Runtime.getRuntime().availableProcessors() - 1;
public static void main(String[] args) throws IOException, InterruptedException {
AtomicLong magicSeed = new AtomicLong(0);
AtomicLong totalAttempts = new AtomicLong(0);
AtomicLong maxCardinality = new AtomicLong(0);
long start = System.currentTimeMillis();
System.out.println("Searching for perfect hash seed for " + DESIRED_SLOTS + " slots");
// Figure out encoding for all possible temperature values (1999 total)
Map<Long, Short> decodeTemperatureMap = new HashMap<>();
for (short i = -999; i <= 999; i++) {
long word = 0;
int shift = 0;
if (i < 0) {
word |= ((long) '-') << shift;
shift += 8;
}
if (Math.abs(i) >= 100) {
int hh = Math.abs(i) / 100;
int tt = (Math.abs(i) - hh * 100) / 10;
word |= ((long) (hh + '0')) << shift;
shift += 8;
word |= ((long) (tt + '0')) << shift;
}
else {
int tt = Math.abs(i) / 10;
// convert to ascii
word |= ((long) (tt + '0')) << shift;
}
shift += 8;
word |= ((long) '.') << shift;
shift += 8;
int uu = Math.abs(i) % 10;
word |= ((long) (uu + '0')) << shift;
// 31302e3000000000
decodeTemperatureMap.put(word, i);
}
ExecutorService executor = Executors.newFixedThreadPool(N_THREADS);
RandomGeneratorFactory factory = RandomGeneratorFactory.of("L64X256MixRandom");
Runnable search = () -> {
// Brute force to find seed:
// generate a cryptographically secure random seed
RandomGenerator rand;
try {
byte[] seed = new byte[16];
SecureRandom.getInstanceStrong().nextBytes(seed);
rand = factory.create(ByteBuffer.wrap(seed).getLong());
System.out.println(Thread.currentThread().getName() + " | Using seed: " + rand.nextLong());
}
catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
int max = 0;
int attempts = 0;
while (true) {
BitSet bs = new BitSet(DESIRED_SLOTS);
var seed = rand.nextLong();
seed |= 0b1; // make sure it's odd
for (var word : decodeTemperatureMap.keySet()) {
var h = (word * seed) & ~(1L << 63);
var pos = (int) (h % DESIRED_SLOTS);
bs.set(pos);
}
var c = bs.cardinality();
if (c == decodeTemperatureMap.size()) {
System.out.println("FOUND seed: " + seed + " cardinality: " + c + " max cardinality: " + max);
magicSeed.set(seed);
return;
}
max = Math.max(max, c);
if (attempts % 100_000 == 0) {
if (magicSeed.get() != 0)
return;
int finalMax = max;
long currentMaxCardinality = maxCardinality.updateAndGet(currentMax -> Math.max(currentMax, finalMax));
long currentTotalAttempts = totalAttempts.addAndGet(100_000);
if (Thread.currentThread().getName().endsWith("-1"))
System.out.println(Thread.currentThread().getName() + " | max cardinality: " + currentMaxCardinality + " attempts: "
+ String.format("%,d", currentTotalAttempts));
}
attempts++;
}
};
for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) {
executor.submit(search);
}
// Wait for the search to complete
executor.shutdown();
executor.awaitTermination(1, TimeUnit.DAYS);
short[] TEMPERATURES = new short[DESIRED_SLOTS];
long seed = magicSeed.get();
decodeTemperatureMap.entrySet().stream().forEach(e -> {
var word = e.getKey();
var h = (word * seed) & ~(1L << 63);
var pos = (int) (h % DESIRED_SLOTS);
if (TEMPERATURES[pos] != 0)
throw new RuntimeException("collision at " + pos);
TEMPERATURES[pos] = e.getValue();
});
System.out.println("SUCCESS seed: " + seed + " total attempts: " + totalAttempts.get());
try {
File file = new File("seeds.txt");
file.delete();
file.createNewFile();
// Write the seed to seeds.txt
FileWriter myWriter = new FileWriter("seeds.txt");
myWriter.write(Long.toString(seed));
myWriter.write("\n");
myWriter.close();
}
catch (IOException e) {
throw new RuntimeException(e);
}
System.out.println("Search took " + ((System.currentTimeMillis() - start) / 1000) + "s");
}
}