Playground/1brc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Limit names to 100 bytes

Browse Source
This commit is contained in:
Marko Topolnik 2024-01-05 23:03:28 +01:00 committed by Gunnar Morling
parent a094d07925
commit e09cb7deea
3 changed files with 50 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/dev/morling/onebrc/CreateMeasurements3.java
View File

@ -21,6 +21,7 @@ import java.io.FileReader;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.io.StringReader; import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet; import java.util.HashSet;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
@ -113,6 +114,9 @@ public class CreateMeasurements3 {
while (names.contains(name)) { while (names.contains(name)) {
name = name.substring(1) + readNonSpace(nameSource); name = name.substring(1) + readNonSpace(nameSource);
} }
while (name.getBytes(StandardCharsets.UTF_8).length > 100) {
name = name.substring(0, name.length() - 1);
}
if (name.indexOf(';') != -1) { if (name.indexOf(';') != -1) {
throw new Exception("Station name contains a semicolon!"); throw new Exception("Station name contains a semicolon!");
} }

2
src/test/resources/samples/measurements-complex-utf8.out
View File

@ -1 +1 @@
{ Al=18.5/18.5/18.5, FraijanesOuricur=29.3/29.3/29.3, akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne=19.2/19.2/19.2, arh SisaiOulad AyyadAhirauliyāVilla=7.9/7.9/7.9, arpā=23.7/23.7/23.7, diqabadShahr-e Ko=14.0/14.0/14.0, einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo=11.9/11.9/11.9, e‘utPic=15.9/15.9/15.9, hleyHornchurchLiepājaActonKatiBouiraPort=10.7/10.7/10.7, iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd=18.6/18.6/18.6, ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai=9.8/9.8/9.8, ingtonAlcalá de HenaresTarantoKhowyUr=18.8/18.8/18.8, kohā=28.1/28.1/28.1, l ḨazmĀlampur GonpuraKr=8.0/8.0/8.0, lţiWłocławekMaumere=20.4/20.4/20.4, molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú=5.6/5.6/5.6, n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para=17.2/17.2/17.2, nBanī SuwayfCollege StationBà Rị=-4.7/-4.7/-4.7, ntevedraChilliwackBuuhoodleFarmin=17.5/17.5/17.5, onGwadarCambambeVictoriasPinamalayanKatiolaC=12.0/12.0/12.0, riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam=12.8/12.8/12.8, rlooKaikkudiBernalKamagayaNor=31.8/31.8/31.8, saNāsri=11.8/11.8/11.8, uOn=17.5/17.5/17.5, ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh=2.6/2.6/2.6, yuMekeleGon=15.7/15.7/15.7, ālaV=26.0/26.0/26.0, ž=12.3/12.3/12.3, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ=7.1/10.1/13.0, ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè=11.1/17.2/23.3} {B=8.9/8.9/8.9, C=38.9/38.9/38.9, CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa=3.0/3.0/3.0, ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS=23.4/23.4/23.4, MirnaPehčevoRopažiGus=16.7/16.7/16.7, PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb=17.5/17.5/17.5, TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka=10.6/10.6/10.6, aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS=25.4/25.4/25.4, burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak=21.5/21.5/21.5, cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS=11.2/11.2/11.2, eLafayetteAsh Shaţ=14.2/14.2/14.2, en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi=11.9/11.9/11.9, epé=28.2/28.2/28.2, hanVarkkallaiPort LokoD=10.9/10.9/10.9, iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM=13.4/13.4/13.4, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul=22.5/22.5/22.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl=11.5/11.5/11.5, igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul=18.5/18.5/18.5, inhoSökeDordrechtPoáLaloG=13.1/13.1/13.1, iudad Melchor MúzquizQuinhámelDa=40.5/40.5/40.5, l ‘=14.6/14.6/14.6, lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait=22.8/22.8/22.8, liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque=12.4/12.4/12.4, lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe=22.5/22.5/22.5, m el Bo=14.6/14.6/14.6, mazunchaleZrenjaninFouchanaSurtPanč=6.7/6.7/6.7, ngoDübendorfC=11.7/11.7/11.7, nt-A=9.2/9.2/9.2, ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon=14.6/14.6/14.6, oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe=17.3/17.3/17.3, oGumlāSamā’=14.9/14.9/14.9, os Reyes de SalgadoCinisello BalsamoKashibaH=20.0/20.0/20.0, picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy=13.0/13.0/13.0, raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat=13.5/13.5/13.5, rhamDera Ghazi KhanMiyazakiBhātpār=21.3/21.3/21.3, rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan=10.9/10.9/10.9, skişeh=12.9/12.9/12.9, venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC=1.9/1.9/1.9, y-le-MoutierSant’ArpinoPljevljaRo=0.8/0.8/0.8, ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK=6.0/6.0/6.0, üSosnowiecTanauanMya=18.4/18.4/18.4, ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E=19.4/19.4/19.4, āng=15.7/15.7/15.7, ġFis=9.6/9.6/9.6, ‘AqabahPembaNowgongQu=12.9/12.9/12.9}

77
src/test/resources/samples/measurements-complex-utf8.txt
View File

@ -1,32 +1,45 @@
kohā;28.1 aniCartagoEṭ ṬīraTemerinCormeilles-en-ParisisZawyat ech CheïkhS;25.4
lţiWłocławekMaumere;20.4 picuíbaJhang CityTepicJayapuraRio BrancoToyamaFangtingSanandajDelhi CantonmentLinghaiShorāpurToy;13.0
Al;18.5 lhuleuTacurongNavapolatskPiscoDera Ismail KhanLabéAltamiraCavite CityYevpatoriiaTait;22.8
arh SisaiOulad AyyadAhirauliyāVilla;7.9 āng;15.7
molaLimonadeSheboyganCasas AdobesSt. CharlesRhonddaSatyamangalamKemptenVilla del RosarioSanlú;5.6 hanVarkkallaiPort LokoD;10.9
hleyHornchurchLiepājaActonKatiBouiraPort;10.7 eLafayetteAsh Shaţ;14.2
onGwadarCambambeVictoriasPinamalayanKatiolaC;12.0 ‘AqabahPembaNowgongQu;12.9
saNāsri;11.8 inhoSökeDordrechtPoáLaloG;13.1
nBanī SuwayfCollege StationBà Rị;-4.7 skişeh;12.9
ntevedraChilliwackBuuhoodleFarmin;17.5 rhamDera Ghazi KhanMiyazakiBhātpār;21.3
rlooKaikkudiBernalKamagayaNor;31.8 igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPodl;11.5
ySarhAn NuhūdKotmaleJoyabajDearbornCentennialKoszalinTajimiTh;2.6 igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkopunGornji PetrovciRibnicaKon TumŠavnikPoul;18.5
FraijanesOuricur;29.3 igButeboJuršinciKoaniImdinaNova VasDestrnikVarvarinSkomunGornji PetrovciRibnicaKon TumŠavnikPoul;22.5
iaBent JbaïlMažeikiaiWest Little RiverMukdahanAd;18.6 B;8.9
ālaV;26.0 C;38.9
ž;12.3 nt-A;9.2
l ḨazmĀlampur GonpuraKr;8.0 y-le-MoutierSant’ArpinoPljevljaRo;0.8
einachCoalcomán de Vázquez PallaraesGorokaSaint-LambertChamalièresAcatlán de OsorioOpuwoAldo Bo;11.9 oGumlāSamā’;14.9
ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;23.3 os Reyes de SalgadoCinisello BalsamoKashibaH;20.0
ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;13.0 m el Bo;14.6
ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀè;11.1 mazunchaleZrenjaninFouchanaSurtPanč;6.7
ḨĀáèá‘òìãüyüçḨĀáèáòìãüyüçḨĀáèāáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyüçḨĀáèáòìãüyçḨĀ;7.1 ġFis;9.6
uOn;17.5 epé;28.2
yuMekeleGon;15.7 ālSongnimSanto TomasKoiduHoshangābādOpoleNovocheboksarskArarasKhannaPunoKoforiduaAhmadpur E;19.4
riHòa BìnhKuopioRibeirão PiresOndjivaPort-de-PaixShuixiWulingSam;12.8 iudad Melchor MúzquizQuinhámelDa;40.5
e‘utPic;15.9 ChesterLobnyaSan LeandroHemeiSolweziGrand BourgKaliboS;23.4
akiBüyükçekmeceKhomeynī ShahrCarmenMcKinne;19.2 cotánSan Ramón de la Nueva OránWausauGbaweTailaiRochester HillsVilla ElisaToba TekS;11.2
arpā;23.7 raKielSibuYatoParanáSanta ClaraYamagataKatihārBeykozImperat;13.5
ibertéChamthaHuatabampoChichesterSmolyanIsingiroSwakopmundMontgomery VillageHoma BayFai;9.8 l ‘;14.6
n HarroudaKashiwaraŢūkhDimbokroPassaicVeenendaalAgía Para;17.2 TanjungpinangKasselHaldiaLuxorLạng SơnAt TājīTaraka;10.6
ingtonAlcalá de HenaresTarantoKhowyUr;18.8 MirnaPehčevoRopažiGus;16.7
diqabadShahr-e Ko;14.0 üSosnowiecTanauanMya;18.4
ngoDübendorfC;11.7
liLoretoPlacentiaAliso ViejoChomaPen-y-Bont ar OgwrCojutepeque;12.4
burgazAl ḨawīyahSalamancaMbanza KongoNchelengeZhangaözenTurbatMatiMangghystaūMalak;21.5
iCoahuitlánRabatJahāngīrpur SālkhaniCamUniversity of California-Santa BarbaraSerravalleTelkathuM;13.4
lioúpoliBarahonaHoPhuketLe BardoBuena ParkKayesChampigny-sur-MarneHaskovoChathamBatleyEsteioRe;22.5
PototanSahuayo de MorelosBambergMosigkauFrancisco BeltrãoJelenia GóraTelêmaco Borb;17.5
CabindaKermānZunhuaRochesterValenzuelaOrūmīyehWugangShuangqiaoTshikapa;3.0
venGaopingDunhuaAz Zarqā’SylhetKaihuaCaerdyddJāmnagarFuyuanGayaFlorianópolisC;1.9
ntington StationKampong SpeuKakataMoschátoBressoVentspilsSaint-CloudTamboSidi Smai’ilDandenon;14.6
rugarhVerāvalAlagoinhasEdremitBandırmaSalavatGandajikaLucapaLeesburgTamaRas Tan;10.9
oCanagatanHelsinkiJabalpurProvidenceRuchengNizhniy NovgorodAhvāzJeparaShaoyangComayagüe;17.3
ça PaulistaDarmstadtZhengdingPindamonhangabaEnschedeGirónUttarpāraHeidelbergK;6.0
en IslandKota BharuCiudad López MateosCelayaVinhDuyunLos Mochis‘AjmānNyalaLarkanaWichitaNishi;11.9