diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a4658307c3c..1ff7178e9d8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -115,8 +115,9 @@ jobs: fi mkdir benchmark && cd benchmark - ../source/tests/benchmarks/benchmark.sh \ + ../source/benchmarks/cli/benchmark.sh \ ${SUITE_FLAG} ${DEBUG_FLAG} \ + --admin \ --baseline-cli "git" --cli "${GIT2_CLI}" --name libgit2 \ --json benchmarks.json --flamegraph --zip benchmarks.zip shell: bash diff --git a/tests/benchmarks/README.md b/benchmarks/cli/README.md similarity index 100% rename from tests/benchmarks/README.md rename to benchmarks/cli/README.md diff --git a/tests/benchmarks/_script/flamegraph/README.md b/benchmarks/cli/_script/flamegraph/README.md similarity index 100% rename from tests/benchmarks/_script/flamegraph/README.md rename to benchmarks/cli/_script/flamegraph/README.md diff --git a/tests/benchmarks/_script/flamegraph/aix-perf.pl b/benchmarks/cli/_script/flamegraph/aix-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/aix-perf.pl rename to benchmarks/cli/_script/flamegraph/aix-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/difffolded.pl b/benchmarks/cli/_script/flamegraph/difffolded.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/difffolded.pl rename to benchmarks/cli/_script/flamegraph/difffolded.pl diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt b/benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace-stacks.txt rename to benchmarks/cli/_script/flamegraph/example-dtrace-stacks.txt diff --git a/tests/benchmarks/_script/flamegraph/example-dtrace.svg b/benchmarks/cli/_script/flamegraph/example-dtrace.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-dtrace.svg rename to benchmarks/cli/_script/flamegraph/example-dtrace.svg diff --git a/tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz b/benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf-stacks.txt.gz rename to benchmarks/cli/_script/flamegraph/example-perf-stacks.txt.gz diff --git a/tests/benchmarks/_script/flamegraph/example-perf.svg b/benchmarks/cli/_script/flamegraph/example-perf.svg similarity index 100% rename from tests/benchmarks/_script/flamegraph/example-perf.svg rename to benchmarks/cli/_script/flamegraph/example-perf.svg diff --git a/tests/benchmarks/_script/flamegraph/files.pl b/benchmarks/cli/_script/flamegraph/files.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/files.pl rename to benchmarks/cli/_script/flamegraph/files.pl diff --git a/tests/benchmarks/_script/flamegraph/flamegraph.pl b/benchmarks/cli/_script/flamegraph/flamegraph.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/flamegraph.pl rename to benchmarks/cli/_script/flamegraph/flamegraph.pl diff --git a/tests/benchmarks/_script/flamegraph/jmaps b/benchmarks/cli/_script/flamegraph/jmaps similarity index 100% rename from tests/benchmarks/_script/flamegraph/jmaps rename to benchmarks/cli/_script/flamegraph/jmaps diff --git a/tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl b/benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/pkgsplit-perf.pl rename to benchmarks/cli/_script/flamegraph/pkgsplit-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/range-perf.pl b/benchmarks/cli/_script/flamegraph/range-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/range-perf.pl rename to benchmarks/cli/_script/flamegraph/range-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/record-test.sh b/benchmarks/cli/_script/flamegraph/record-test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/record-test.sh rename to benchmarks/cli/_script/flamegraph/record-test.sh diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-aix.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-aix.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-bpftrace.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-bpftrace.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py b/benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-chrome-tracing.py rename to benchmarks/cli/_script/flamegraph/stackcollapse-chrome-tracing.py diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-elfutils.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-elfutils.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-faulthandler.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-faulthandler.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-gdb.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-gdb.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-go.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-go.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-go.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-go.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ibmjava.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-ibmjava.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-instruments.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-instruments.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-java-exceptions.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-java-exceptions.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-jstack.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-jstack.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-ljp.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-ljp.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf-sched.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf-sched.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-perf.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-perf.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-pmc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-pmc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-recursive.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-recursive.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk b/benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-sample.awk rename to benchmarks/cli/_script/flamegraph/stackcollapse-sample.awk diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-stap.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-stap.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vsprof.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vsprof.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune-mc.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune-mc.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-vtune.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-vtune.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl b/benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-wcp.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse-wcp.pl diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php b/benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse-xdebug.php rename to benchmarks/cli/_script/flamegraph/stackcollapse-xdebug.php diff --git a/tests/benchmarks/_script/flamegraph/stackcollapse.pl b/benchmarks/cli/_script/flamegraph/stackcollapse.pl similarity index 100% rename from tests/benchmarks/_script/flamegraph/stackcollapse.pl rename to benchmarks/cli/_script/flamegraph/stackcollapse.pl diff --git a/tests/benchmarks/_script/flamegraph/test.sh b/benchmarks/cli/_script/flamegraph/test.sh similarity index 100% rename from tests/benchmarks/_script/flamegraph/test.sh rename to benchmarks/cli/_script/flamegraph/test.sh diff --git a/tests/benchmarks/benchmark.sh b/benchmarks/cli/benchmark.sh similarity index 90% rename from tests/benchmarks/benchmark.sh rename to benchmarks/cli/benchmark.sh index e82eaabfad1..34728c6a283 100755 --- a/tests/benchmarks/benchmark.sh +++ b/benchmarks/cli/benchmark.sh @@ -6,7 +6,7 @@ set -eo pipefail # parse the command line # -usage() { echo "usage: $(basename "$0") [--cli ] [--name ] [--baseline-cli ] [--suite ] [--json ] [--flamegraph] [--zip ] [--verbose] [--debug]"; } +usage() { echo "usage: $(basename "$0") [--cli ] [--name ] [--baseline-cli ] [--suite ] [--admin] [--json ] [--flamegraph] [--zip ] [--verbose] [--debug]"; } TEST_CLI="git" TEST_CLI_NAME= @@ -16,6 +16,7 @@ JSON_RESULT= FLAMEGRAPH= ZIP_RESULT= OUTPUT_DIR= +ADMIN= VERBOSE= DEBUG= NEXT= @@ -58,6 +59,8 @@ for a in "$@"; do NEXT="suite" elif [[ "${a}" == "-s"* ]]; then SUITE="${a/-s/}" + elif [ "${a}" == "--admin" ]; then + ADMIN=1 elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then VERBOSE=1 elif [ "${a}" == "--debug" ]; then @@ -126,7 +129,7 @@ cli_commit() { BUILD_OPTIONS=$("$1" version --build-options) fi - echo "${BUILD_OPTIONS}" | grep '^built from commit: ' | sed -e 's/^built from commit: //' + echo "${BUILD_OPTIONS}" | { grep '^built from commit: ' || echo "unknown"; } | sed -e 's/^built from commit: //' } TEST_CLI_NAME=$(basename "${TEST_CLI}") @@ -162,9 +165,9 @@ echo "########################################################################## echo "" if [ "${BASELINE_CLI}" != "" ]; then - echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION})" + echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION} ${BASELINE_CLI_COMMIT:0:7})" fi -echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION})" +echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION} ${TEST_CLI_COMMIT:0:7})" echo "" BENCHMARK_DIR=${BENCHMARK_DIR:=$(dirname "$0")} @@ -209,6 +212,8 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do continue fi + TEST_DESCRIPTION=$(( grep '^# Description: ' "${TEST_PATH}" || test $? = 1 ) | sed -e "s/^# Description: //") + ANY_FOUND=1 TEST_NAME="${TEST_FILE/__/::}" @@ -223,6 +228,10 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do SHOW_OUTPUT="--show-output" fi + if [ "${ADMIN}" = "1" ]; then + ALLOW_ADMIN="--admin" + fi + OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out" ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err" JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json" @@ -230,7 +239,7 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do FAILED= { - ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}"; + ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${ALLOW_ADMIN} ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}"; FAILED=$? } || true @@ -272,7 +281,7 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do fi # add our metadata to the hyperfine json result - jq ". |= { \"name\": \"${TEST_NAME}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}" + jq ". |= { \"name\": \"${TEST_NAME}\", \"description\": \"${TEST_DESCRIPTION}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}" # run with flamegraph output if requested if [ "${FLAMEGRAPH}" ]; then @@ -311,6 +320,8 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do ANY_FAILED=1 fi fi + else + echo "" fi done diff --git a/tests/benchmarks/benchmark_helpers.sh b/benchmarks/cli/benchmark_helpers.sh similarity index 93% rename from tests/benchmarks/benchmark_helpers.sh rename to benchmarks/cli/benchmark_helpers.sh index e5babf9c819..f78aabc09f8 100644 --- a/tests/benchmarks/benchmark_helpers.sh +++ b/benchmarks/cli/benchmark_helpers.sh @@ -7,11 +7,12 @@ set -eo pipefail # command-line parsing # -usage() { echo "usage: $(basename "$0") [--cli ] [--baseline-cli ] [--output-style