diff --git a/scripts/compare.sh b/scripts/compare.sh index 0d58370..7bbaee9 100755 --- a/scripts/compare.sh +++ b/scripts/compare.sh @@ -71,8 +71,8 @@ echo "Benchmark regression check" echo "==========================" echo "" -# Relative checks (ferrflow benchmarks only) -for key in $(jq -r '.benchmarks | keys[]' "$LATEST" | grep '\-ferrflow-'); do +# Relative checks (ferrflow binary under test only) +for key in $(jq -r '.benchmarks | keys[]' "$LATEST" | grep -- '-ferrflow-binary-'); do new_median=$(jq -r ".benchmarks[\"$key\"].median_ms" "$LATEST") old_median=$(jq -r ".benchmarks[\"$key\"].median_ms // empty" "$BASELINE" 2>/dev/null || echo "") diff --git a/scripts/run.sh b/scripts/run.sh index 1d6c9ce..293f052 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -186,22 +186,14 @@ measure_memory() { fi } -get_npm_binary_size() { - local pkg="$1" - local tmp_dir bin - tmp_dir=$(mktemp -d) - ( - cd "$tmp_dir" - npm init -y &>/dev/null - npm install --save "$pkg" &>/dev/null 2>&1 - ) - bin=$(find "$tmp_dir/node_modules" -type f -path "*/@$pkg/*/bin/$pkg" 2>/dev/null | head -1) +get_binary_size() { + local bin + bin=$(command -v "$1" 2>/dev/null || true) if [[ -n "$bin" && -f "$bin" ]]; then - du -m "$bin" | awk '{printf "%.1f", $1}' + stat -L -c %s "$bin" | awk '{printf "%.1f", $1/1048576}' else echo "N/A" fi - rm -rf "$tmp_dir" } # Get npm package install size in MB @@ -306,7 +298,7 @@ for tool in $TOOLS; do echo "$npm_size" > "$RAW_DIR/${tool}-npm-size.txt" fi if [[ "$method" == "binary" && "$tool" == "ferrflow" ]]; then - bin_size=$(get_npm_binary_size ferrflow) + bin_size=$(get_binary_size "$local_bin") echo "$bin_size" > "$RAW_DIR/ferrflow-binary-size.txt" fi diff --git a/tests/compare.bats b/tests/compare.bats index 5b7f70e..5c4692a 100644 --- a/tests/compare.bats +++ b/tests/compare.bats @@ -157,3 +157,15 @@ EOF [ "$status" -eq 1 ] [[ "$output" == *"Usage"* ]] } + +@test "ignores npm and docker ferrflow regressions" { + make_json "$TMPDIR/baseline.json" \ + '"single-ferrflow-npm-check": {"median_ms": 100.0, "stddev_ms": 1.0, "memory_mb": "N/A"}, "single-ferrflow-docker-check": {"median_ms": 100.0, "stddev_ms": 1.0, "memory_mb": "N/A"}' + make_json "$TMPDIR/latest.json" \ + '"single-ferrflow-npm-check": {"median_ms": 300.0, "stddev_ms": 1.0, "memory_mb": "N/A"}, "single-ferrflow-docker-check": {"median_ms": 300.0, "stddev_ms": 1.0, "memory_mb": "N/A"}' + + run bash "$SCRIPT_DIR/compare.sh" "$TMPDIR/baseline.json" "$TMPDIR/latest.json" + [ "$status" -eq 0 ] + [[ "$output" != *"single-ferrflow-npm-check"* ]] + [[ "$output" == *"All benchmarks within thresholds"* ]] +}