From c92c88b9fbea06422bbadeafe237c134361ecb88 Mon Sep 17 00:00:00 2001 From: Jason Nochlin <91577+hundredwatt@users.noreply.github.com> Date: Wed, 10 Jan 2024 02:05:16 -0700 Subject: [PATCH] evaluate2.sh improvements - leaderboard, default SDK * reset the JDK to the default (21.0.1-open) when no prepare script is provided * leaderboard improvements - sorting and content * run sdk install once at the beginning of the script for all the SDKs detected in any of the evaluated prepare scripts * remove unnecessary code and tweak doc comments * one more nit * Don't print rankings values when only 1 fork is being evaluated * It's been a few hours, so I now have some more rate limit :) --------- Co-authored-by: Jason Nochlin --- evaluate2.sh | 105 ++++++++++++++++++++++++++++++++++++++++++----- github_users.txt | 50 ++++++++++++++++++++++ pom.xml | 1 + 3 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 github_users.txt diff --git a/evaluate2.sh b/evaluate2.sh index ac5ab8f..ab54d7a 100755 --- a/evaluate2.sh +++ b/evaluate2.sh @@ -34,6 +34,8 @@ RED='\033[0;31m' BOLD_YELLOW='\033[1;33m' RESET='\033[0m' # No Color +DEFAULT_JAVA_VERSION="21.0.1-open" + function check_command_installed { if ! [ -x "$(command -v $1)" ]; then echo "Error: $1 is not installed." >&2 @@ -45,6 +47,35 @@ check_command_installed java check_command_installed hyperfine check_command_installed jq +## SDKMAN Setup +# 1. Custom check for sdkman installed; not sure why check_command_installed doesn't detect it properly +if [ ! -f "$HOME/.sdkman/bin/sdkman-init.sh" ]; then + echo "Error: sdkman is not installed." >&2 + exit 1 +fi + +# 2. Init sdkman in this script +source "$HOME/.sdkman/bin/sdkman-init.sh" + +# 3. make sure the default java version is installed +if [ ! -d "$HOME/.sdkman/candidates/java/$DEFAULT_JAVA_VERSION" ]; then + echo "+ sdk install java $DEFAULT_JAVA_VERSION" + sdk install java $DEFAULT_JAVA_VERSION +fi + +# 4. Install missing SDK java versions in any of the prepare_*.sh scripts for the provided forks +for fork in "$@"; do + if [ -f "./prepare_$fork.sh" ]; then + grep -h "^sdk use" "./prepare_$fork.sh" | cut -d' ' -f4 | while read -r version; do + if [ ! -d "$HOME/.sdkman/candidates/java/$version" ]; then + echo "+ sdk install java $version" + sdk install java $version + fi + done + fi +done +## END - SDKMAN Setup + # Check if SMT is enabled (we want it disabled) if [ -f "/sys/devices/system/cpu/smt/active" ]; then if [ "$(cat /sys/devices/system/cpu/smt/active)" != "0" ]; then @@ -88,6 +119,9 @@ for fork in "$@"; do if [ -f "./prepare_$fork.sh" ]; then echo "+ source ./prepare_$fork.sh" source "./prepare_$fork.sh" + else + echo "+ sdk use java $DEFAULT_JAVA_VERSION" + sdk use java $DEFAULT_JAVA_VERSION fi # Optional additional build steps @@ -151,8 +185,13 @@ for fork in "$@"; do done echo "" -# Leaderboard +## Leaderboard - prints the leaderboard in Markdown table format echo -e "${BOLD_WHITE}Leaderboard${RESET}" + +# 1. Create a temp file to store the leaderboard entries +leaderboard_temp_file=$(mktemp) + +# 2. Process each fork and append the 1-line entry to the temp file for fork in "$@"; do # skip reporting results for failed forks if [[ " ${failed[@]} " =~ " ${fork} " ]]; then @@ -161,12 +200,6 @@ for fork in "$@"; do trimmed_mean=$(jq -r '.results[0].times | .[1:-1] | add / length' $fork-$filetimestamp-timing.json) - # Read java version from prepare_$fork.sh if it exists - java_version="unknown" - if [ -f "./prepare_$fork.sh" ]; then - java_version=$(grep "sdk use java" ./prepare_$fork.sh | cut -d' ' -f4) - fi - # trimmed_mean is in seconds # Format trimmed_mean as MM::SS.mmm # using bc @@ -175,13 +208,63 @@ for fork in "$@"; do trimmed_mean_ms=$(echo "($trimmed_mean - $trimmed_mean_minutes * 60 - $trimmed_mean_seconds) * 1000 / 1" | bc) trimmed_mean_formatted=$(printf "%02d:%02d.%03d" $trimmed_mean_minutes $trimmed_mean_seconds $trimmed_mean_ms) - # var result = String.format("%02d:%02d.%.0f", mean.toMinutesPart(), mean.toSecondsPart(), (double) mean.toNanosPart() / 1_000_000); - # var author = actualFile.replace(".out", "") - # System.out.println(String.format("\n| | %s| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_%s.java)| 21.0.1-open | [%s](https://github.com/%s)|", result, author, author, author)); + # Get Github user's name from public Github API (rate limited after ~50 calls, so results are cached in github_users.txt) + set +e + github_user__name=$(grep "^$fork;" github_users.txt | cut -d ';' -f2) + if [ -z "$github_user__name" ]; then + github_user__name=$(curl -s https://api.github.com/users/$fork | jq -r '.name' | tr -d '"') + if [ "$github_user__name" != "null" ]; then + echo "$fork;$github_user__name" >> github_users.txt + else + github_user__name=$fork + fi + fi + set -e - echo "| | $trimmed_mean_formatted| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_$fork.java)| $java_version | [$fork](https://github.com/$fork)|" + # Read java version from prepare_$fork.sh if it exists, otherwise assume 21.0.1-open + java_version="21.0.1-open" + if [ -f "./prepare_$fork.sh" ]; then + java_version=$(grep "sdk use java" ./prepare_$fork.sh | cut -d' ' -f4) + fi + + # Hard-coding the note message for now + notes="" + if [ -f "./additional_build_steps_$fork.sh" ]; then + notes="GraalVM native binary" + fi + + echo -n "$trimmed_mean;" >> $leaderboard_temp_file # for sorting + echo -n "| # " >> $leaderboard_temp_file + echo -n "| $trimmed_mean_formatted " >> $leaderboard_temp_file + echo -n "| [link](https://github.com/gunnarmorling/1brc/blob/main/src/main/java/dev/morling/onebrc/CalculateAverage_$fork.java)" >> $leaderboard_temp_file + echo -n "| $java_version " >> $leaderboard_temp_file + echo -n "| [$github_user__name](https://github.com/$fork) " >> $leaderboard_temp_file + echo -n "| $notes " >> $leaderboard_temp_file + echo "|" >> $leaderboard_temp_file done + +# 3. Sort leaderboard_temp_file by trimmed_mean and remove the sorting column +sort -n $leaderboard_temp_file | cut -d ';' -f 2 > $leaderboard_temp_file.sorted + +# 4. Print the leaderboard echo "" +echo "| # | Result (m:s.ms) | Implementation | JDK | Submitter | Notes |" +echo "|---|-----------------|--------------------|-----|---------------|-----------|" +# If $leaderboard_temp_file.sorted has more than 3 entires, include rankings +if [ $(wc -l < $leaderboard_temp_file.sorted) -gt 3 ]; then + head -n 1 $leaderboard_temp_file.sorted | tr '#' 1 + head -n 2 $leaderboard_temp_file.sorted | tail -n 1 | tr '#' 2 + head -n 3 $leaderboard_temp_file.sorted | tail -n 1 | tr '#' 3 + tail -n+4 $leaderboard_temp_file.sorted | tr '#' ' ' +else + # Don't show rankings + cat $leaderboard_temp_file.sorted | tr '#' ' ' +fi +echo "" + +# 5. Cleanup +rm $leaderboard_temp_file +## END - Leaderboard # Finalize .out files echo "Raw results saved to file(s):" diff --git a/github_users.txt b/github_users.txt new file mode 100644 index 0000000..dfe15ee --- /dev/null +++ b/github_users.txt @@ -0,0 +1,50 @@ +Ujjwalbharti;Ujjwal Bharti +abfrmblr;Abhilash +ags313;ags +anandmattikopp;twohardthings +armandino;Arman Sharif +artpar;Parth Mudgal +asun;Alan Sun +bjhara;Hampus +coolmineman;Cool_Mineman +criccomini;Chris Riccomini +davecom;David Kopec +davery22;Daniel Avery +ddimtirov;Dimitar Dimitrov +ebarlas;Elliot Barlas +entangled90;Carlo +fatroom;Roman Romanchuk +felix19350;Bruno Félix +filiphr;Filip Hrisafov +flippingbits;Stefan Sprenger +fragmede;Samson +gabrielreid;Gabriel Reid +hchiorean;Horia Chiorean +imrafaelmerino;Rafael Merino García +isolgpus;Jamie Stansfield +iziamos;John Ziamos +jgrateron;Jairo Graterón +jotschi;Johannes Schüth +kevinmcmurtrie;Kevin McMurtrie +kgeri;Gergely Kiss +khmarbaise;Karl Heinz Marbaise +kuduwa-keshavram;Keshavram Kuduwa +merykitty;Quan Anh Mai +moysesb;Moysés Borges Furtado +mudit-saxena;Mudit Saxena +nstng;Nils Semmelrock +obourgain;Olivier Bourgain +padreati;Aurelian Tutuianu +palmr;Nick Palmer +rby;Ramzi Ben Yahya +richardstartin;Richard Startin +spullara;Sam Pullara +royvanrijn;Roy van Rijn +seijikun;Markus Ebner +semotpan;Serghei Motpan +thomaswue;Thomas Wuerthinger +truelive;Roman Schweitzer +twobiers;Tobi +yavuztas;Yavuz Tas +yehwankim23;김예환 Ye-Hwan Kim (Sam) +hundredwatt;Jason Nochlin diff --git a/pom.xml b/pom.xml index 61cb80f..a74c588 100644 --- a/pom.xml +++ b/pom.xml @@ -206,6 +206,7 @@ **/measurements*.txt **/measurements*.out out_expected.txt + github_users.txt