evaluate2.sh improvements - leaderboard, default SDK

* reset the JDK to the default (21.0.1-open) when no prepare script is provided

* leaderboard improvements - sorting and content

* run sdk install once at the beginning of the script for all the SDKs detected in any of the evaluated prepare scripts

* remove unnecessary code and tweak doc comments

* one more nit

* Don't print rankings values when only 1 fork is being evaluated

* It's been a few hours, so I now have some more rate limit :)

---------

Co-authored-by: Jason Nochlin <hundredwatt@users.noreply.github.com>

evaluate2.sh

@@ -34,6 +34,8 @@ RED='\033[0;31m'
 BOLD_YELLOW='\033[1;33m'
 RESET='\033[0m' # No Color
 
+DEFAULT_JAVA_VERSION="21.0.1-open"
+
 function check_command_installed {
   if ! [ -x "$(command -v $1)" ]; then
     echo "Error: $1 is not installed." >&2
@@ -45,6 +47,35 @@ check_command_installed java
 check_command_installed hyperfine
 check_command_installed jq
 
+## SDKMAN Setup
+# 1. Custom check for sdkman installed; not sure why check_command_installed doesn't detect it properly
+if [ ! -f "$HOME/.sdkman/bin/sdkman-init.sh" ]; then
+    echo "Error: sdkman is not installed." >&2
+    exit 1
+fi
+
+# 2. Init sdkman in this script
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+
+# 3. make sure the default java version is installed
+if [ ! -d "$HOME/.sdkman/candidates/java/$DEFAULT_JAVA_VERSION" ]; then
+  echo "+ sdk install java $DEFAULT_JAVA_VERSION"
+  sdk install java $DEFAULT_JAVA_VERSION
+fi
+
+# 4. Install missing SDK java versions in any of the prepare_*.sh scripts for the provided forks
+for fork in "$@"; do
+  if [ -f "./prepare_$fork.sh" ]; then
+    grep -h "^sdk use" "./prepare_$fork.sh" | cut -d' ' -f4 | while read -r version; do
+      if [ ! -d "$HOME/.sdkman/candidates/java/$version" ]; then
+        echo "+ sdk install java $version"
+        sdk install java $version
+      fi
+    done
+  fi
+done
+## END - SDKMAN Setup
+
 # Check if SMT is enabled (we want it disabled)
 if [ -f "/sys/devices/system/cpu/smt/active" ]; then
   if [ "$(cat /sys/devices/system/cpu/smt/active)" != "0" ]; then
@@ -88,6 +119,9 @@ for fork in "$@"; do
   if [ -f "./prepare_$fork.sh" ]; then
     echo "+ source ./prepare_$fork.sh"
     source "./prepare_$fork.sh"
+  else
+    echo "+ sdk use java $DEFAULT_JAVA_VERSION"
+    sdk use java $DEFAULT_JAVA_VERSION
   fi
 
   # Optional additional build steps
@@ -151,8 +185,13 @@ for fork in "$@"; do
 done
 echo ""
 
-# Leaderboard
+## Leaderboard - prints the leaderboard in Markdown table format
 echo -e "${BOLD_WHITE}Leaderboard${RESET}"
+
+# 1. Create a temp file to store the leaderboard entries
+leaderboard_temp_file=$(mktemp)
+
+# 2. Process each fork and append the 1-line entry to the temp file
 for fork in "$@"; do
   # skip reporting results for failed forks
   if [[ " ${failed[@]} " =~ " ${fork} " ]]; then
@@ -161,12 +200,6 @@ for fork in "$@"; do
 
   trimmed_mean=$(jq -r '.results[0].times | .[1:-1] | add / length' $fork-$filetimestamp-timing.json)
 
-  # Read java version from prepare_$fork.sh if it exists
-  java_version="unknown"
-  if [ -f "./prepare_$fork.sh" ]; then
-    java_version=$(grep "sdk use java" ./prepare_$fork.sh | cut -d' ' -f4)
-  fi
-
   # trimmed_mean is in seconds
   # Format trimmed_mean as MM::SS.mmm
   # using bc
@@ -175,13 +208,63 @@ for fork in "$@"; do
   trimmed_mean_ms=$(echo "($trimmed_mean - $trimmed_mean_minutes * 60 - $trimmed_mean_seconds) * 1000 / 1" | bc)
   trimmed_mean_formatted=$(printf "%02d:%02d.%03d" $trimmed_mean_minutes $trimmed_mean_seconds $trimmed_mean_ms)
 
-  # var result = String.format("%02d:%02d.%.0f", mean.toMinutesPart(), mean.toSecondsPart(), (double) mean.toNanosPart() / 1_000_000);
+  # Get Github user's name from public Github API (rate limited after ~50 calls, so results are cached in github_users.txt)
-  # var author = actualFile.replace(".out", "")
+  set +e
-  # System.out.println(String.format("\n|   |        %s| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_%s.java)| 21.0.1-open | [%s](https://github.com/%s)|", result, author, author, author));
+  github_user__name=$(grep "^$fork;" github_users.txt | cut -d ';' -f2)
+  if [ -z "$github_user__name" ]; then
+    github_user__name=$(curl -s https://api.github.com/users/$fork | jq -r '.name' | tr -d '"')
+    if [ "$github_user__name" != "null" ]; then
+      echo "$fork;$github_user__name" >> github_users.txt
+    else
+      github_user__name=$fork
+    fi
+  fi
+  set -e
 
-  echo "|   |        $trimmed_mean_formatted| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_$fork.java)| $java_version | [$fork](https://github.com/$fork)|"
+  # Read java version from prepare_$fork.sh if it exists, otherwise assume 21.0.1-open
+  java_version="21.0.1-open"
+  if [ -f "./prepare_$fork.sh" ]; then
+    java_version=$(grep "sdk use java" ./prepare_$fork.sh | cut -d' ' -f4)
+  fi
+
+  # Hard-coding the note message for now
+  notes=""
+  if [ -f "./additional_build_steps_$fork.sh" ]; then
+    notes="GraalVM native binary"
+  fi
+
+  echo -n "$trimmed_mean;" >> $leaderboard_temp_file # for sorting
+  echo -n "| # " >> $leaderboard_temp_file
+  echo -n "| $trimmed_mean_formatted " >> $leaderboard_temp_file
+  echo -n "| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_$fork.java)" >> $leaderboard_temp_file
+  echo -n "| $java_version " >> $leaderboard_temp_file
+  echo -n "| [$github_user__name](https://github.com/$fork) " >> $leaderboard_temp_file
+  echo -n "| $notes " >> $leaderboard_temp_file
+  echo "|" >> $leaderboard_temp_file
 done
+
+# 3. Sort leaderboard_temp_file by trimmed_mean and remove the sorting column
+sort -n $leaderboard_temp_file | cut -d ';' -f 2 > $leaderboard_temp_file.sorted
+
+# 4. Print the leaderboard
 echo ""
+echo "| # | Result (m:s.ms) | Implementation     | JDK | Submitter     | Notes     |"
+echo "|---|-----------------|--------------------|-----|---------------|-----------|"
+# If $leaderboard_temp_file.sorted has more than 3 entires, include rankings
+if [ $(wc -l < $leaderboard_temp_file.sorted) -gt 3 ]; then
+  head -n 1 $leaderboard_temp_file.sorted | tr '#' 1
+  head -n 2 $leaderboard_temp_file.sorted | tail -n 1 | tr '#' 2
+  head -n 3 $leaderboard_temp_file.sorted | tail -n 1 | tr '#' 3
+  tail -n+4 $leaderboard_temp_file.sorted | tr '#' ' '
+else
+  # Don't show rankings
+  cat $leaderboard_temp_file.sorted | tr '#' ' '
+fi
+echo ""
+
+# 5. Cleanup
+rm $leaderboard_temp_file
+## END - Leaderboard
 
 # Finalize .out files
 echo "Raw results saved to file(s):"

github_users.txt

@@ -0,0 +1,50 @@
+Ujjwalbharti;Ujjwal Bharti
+abfrmblr;Abhilash
+ags313;ags
+anandmattikopp;twohardthings
+armandino;Arman Sharif
+artpar;Parth Mudgal
+asun;Alan Sun
+bjhara;Hampus
+coolmineman;Cool_Mineman
+criccomini;Chris Riccomini
+davecom;David Kopec
+davery22;Daniel Avery
+ddimtirov;Dimitar Dimitrov
+ebarlas;Elliot Barlas
+entangled90;Carlo
+fatroom;Roman Romanchuk
+felix19350;Bruno Félix
+filiphr;Filip Hrisafov
+flippingbits;Stefan Sprenger
+fragmede;Samson
+gabrielreid;Gabriel Reid
+hchiorean;Horia Chiorean
+imrafaelmerino;Rafael Merino García
+isolgpus;Jamie Stansfield
+iziamos;John Ziamos
+jgrateron;Jairo Graterón
+jotschi;Johannes Schüth
+kevinmcmurtrie;Kevin McMurtrie
+kgeri;Gergely Kiss
+khmarbaise;Karl Heinz Marbaise
+kuduwa-keshavram;Keshavram Kuduwa
+merykitty;Quan Anh Mai
+moysesb;Moysés Borges Furtado
+mudit-saxena;Mudit Saxena
+nstng;Nils Semmelrock
+obourgain;Olivier Bourgain
+padreati;Aurelian Tutuianu
+palmr;Nick Palmer
+rby;Ramzi Ben Yahya
+richardstartin;Richard Startin
+spullara;Sam Pullara
+royvanrijn;Roy van Rijn
+seijikun;Markus Ebner
+semotpan;Serghei Motpan
+thomaswue;Thomas Wuerthinger
+truelive;Roman Schweitzer
+twobiers;Tobi
+yavuztas;Yavuz Tas
+yehwankim23;김예환 Ye-Hwan Kim (Sam)
+hundredwatt;Jason Nochlin

pom.xml

@@ -206,6 +206,7 @@
                 <exclude>**/measurements*.txt</exclude>
                 <exclude>**/measurements*.out</exclude>
                 <exclude>out_expected.txt</exclude>
+                <exclude>github_users.txt</exclude>
               </excludes>
             </configuration>
             <executions>