feat(usable): package gui-host validation snapshot
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Paperclip CTO
276
scripts/qa/run-failure-path-evidence.sh
Executable file
276
scripts/qa/run-failure-path-evidence.sh
Executable file
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -lt 1 ] || [ "$#" -gt 3 ]; then
|
||||
echo "usage: scripts/qa/run-failure-path-evidence.sh <binary-path> [postgres|mysql|all] [output-dir]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
binary_path="$1"
|
||||
scope="${2:-all}"
|
||||
output_dir="${3:-./tmp/failure-path-evidence}"
|
||||
|
||||
if [ ! -x "$binary_path" ]; then
|
||||
echo "error: binary is not executable: $binary_path" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
case "$scope" in
|
||||
postgres|mysql|all) ;;
|
||||
*)
|
||||
echo "error: scope must be one of: postgres, mysql, all" >&2
|
||||
exit 4
|
||||
;;
|
||||
esac
|
||||
|
||||
mkdir -p "$output_dir"
|
||||
|
||||
operator="${QA_OPERATOR:-qa-engineer}"
|
||||
host_name="${QA_HOST_LABEL:-$(hostname 2>/dev/null || echo unknown-host)}"
|
||||
utc_now="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
||||
binary_label="${QA_BINARY_PATH_LABEL:-$binary_path}"
|
||||
binary_version="${QA_BINARY_VERSION:-$("$binary_path" --version 2>/dev/null | head -n 1 || echo unknown-version)}"
|
||||
execution_path="${QA_EXECUTION_PATH:-host}"
|
||||
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
commit_sha="${QA_COMMIT_SHA:-$(git -C "$repo_root" rev-parse HEAD 2>/dev/null || echo unknown-commit)}"
|
||||
branch_name="${QA_BRANCH_NAME:-$(git -C "$repo_root" branch --show-current 2>/dev/null || echo unknown-branch)}"
|
||||
postgres_host="${DBTOOL_POSTGRES_HOST:-127.0.0.1}"
|
||||
postgres_port="${DBTOOL_POSTGRES_PORT:-55432}"
|
||||
mysql_host="${DBTOOL_MYSQL_HOST:-127.0.0.1}"
|
||||
mysql_port="${DBTOOL_MYSQL_PORT:-53306}"
|
||||
postgres_net_port="${DBTOOL_POSTGRES_NET_PORT:-59999}"
|
||||
mysql_net_port="${DBTOOL_MYSQL_NET_PORT:-59998}"
|
||||
|
||||
has_missing_traceability_value() {
|
||||
local value="$1"
|
||||
local placeholder="${2:-}"
|
||||
|
||||
if [ -z "$value" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -n "$placeholder" ] && [ "$value" = "$placeholder" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
contains_secret_leakage() {
|
||||
local raw_output_file="$1"
|
||||
local password_value="$2"
|
||||
local leakage_mode="${3:-contextual}"
|
||||
|
||||
[ -z "$password_value" ] && return 1
|
||||
|
||||
if [ "$leakage_mode" = "exact" ] && grep -Fq "$password_value" "$raw_output_file"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local patterns=(
|
||||
"\"password\":\"$password_value\""
|
||||
"\"password\": \"$password_value\""
|
||||
"password=$password_value"
|
||||
"password: $password_value"
|
||||
"password $password_value"
|
||||
"DBTOOL_PASSWORD=$password_value"
|
||||
":$password_value@"
|
||||
)
|
||||
|
||||
local pattern
|
||||
for pattern in "${patterns[@]}"; do
|
||||
if grep -Fq "$pattern" "$raw_output_file"; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
extract_json_string_field() {
|
||||
local raw_output_file="$1"
|
||||
local field_name="$2"
|
||||
|
||||
sed -nE "s/.*\"${field_name}\"[[:space:]]*:[[:space:]]*\"([^\"]+)\".*/\\1/p" "$raw_output_file" \
|
||||
| head -n 1
|
||||
}
|
||||
|
||||
run_scenario() {
|
||||
local scenario_id="$1"
|
||||
local driver="$2"
|
||||
local expected_kind="$3"
|
||||
local password_value="$4"
|
||||
local leakage_mode="$5"
|
||||
shift 5
|
||||
local args=("$@")
|
||||
|
||||
local raw_output_file="$output_dir/${scenario_id}.out"
|
||||
local markdown_file="$output_dir/${scenario_id}.md"
|
||||
|
||||
(
|
||||
export DBTOOL_PASSWORD="$password_value"
|
||||
set +e
|
||||
"$binary_path" "${args[@]}" >"$raw_output_file" 2>&1
|
||||
local exit_code=$?
|
||||
set -e
|
||||
|
||||
local output_excerpt
|
||||
output_excerpt="$(sed -n '1,120p' "$raw_output_file")"
|
||||
|
||||
local status_value
|
||||
status_value="$(extract_json_string_field "$raw_output_file" "status")"
|
||||
local state_value
|
||||
state_value="$(extract_json_string_field "$raw_output_file" "state")"
|
||||
local kind_value
|
||||
kind_value="$(extract_json_string_field "$raw_output_file" "kind")"
|
||||
local key_output_summary="status=${status_value:-missing}, state=${state_value:-missing}, error.kind=${kind_value:-missing}"
|
||||
|
||||
local leakage_status="not-detected"
|
||||
if contains_secret_leakage "$raw_output_file" "$password_value" "$leakage_mode"; then
|
||||
leakage_status="detected"
|
||||
fi
|
||||
|
||||
local command_text="$binary_label ${args[*]}"
|
||||
local traceability_gaps=()
|
||||
if has_missing_traceability_value "$utc_now"; then
|
||||
traceability_gaps+=("date")
|
||||
fi
|
||||
if has_missing_traceability_value "$operator"; then
|
||||
traceability_gaps+=("operator")
|
||||
fi
|
||||
if has_missing_traceability_value "$host_name" "unknown-host"; then
|
||||
traceability_gaps+=("host")
|
||||
fi
|
||||
if has_missing_traceability_value "$binary_label"; then
|
||||
traceability_gaps+=("binary_path")
|
||||
fi
|
||||
if has_missing_traceability_value "$binary_version" "unknown-version"; then
|
||||
traceability_gaps+=("binary_version")
|
||||
fi
|
||||
if has_missing_traceability_value "$execution_path"; then
|
||||
traceability_gaps+=("execution_path")
|
||||
fi
|
||||
if has_missing_traceability_value "$commit_sha" "unknown-commit"; then
|
||||
traceability_gaps+=("commit_sha")
|
||||
fi
|
||||
if has_missing_traceability_value "$branch_name" "unknown-branch"; then
|
||||
traceability_gaps+=("branch")
|
||||
fi
|
||||
if has_missing_traceability_value "$driver"; then
|
||||
traceability_gaps+=("driver")
|
||||
fi
|
||||
if has_missing_traceability_value "$scenario_id"; then
|
||||
traceability_gaps+=("scenario_id")
|
||||
fi
|
||||
if has_missing_traceability_value "$command_text"; then
|
||||
traceability_gaps+=("command")
|
||||
fi
|
||||
if has_missing_traceability_value "$key_output_summary"; then
|
||||
traceability_gaps+=("key_output")
|
||||
fi
|
||||
|
||||
local traceability_status="complete"
|
||||
local traceability_summary="none"
|
||||
if [ "${#traceability_gaps[@]}" -gt 0 ]; then
|
||||
traceability_status="incomplete"
|
||||
traceability_summary="$(printf '%s, ' "${traceability_gaps[@]}")"
|
||||
traceability_summary="${traceability_summary%, }"
|
||||
fi
|
||||
|
||||
local verdict="fail"
|
||||
if [ "$exit_code" -ne 0 ] \
|
||||
&& grep -Eq '"status"[[:space:]]*:[[:space:]]*"error"' "$raw_output_file" \
|
||||
&& grep -Eq '"state"[[:space:]]*:[[:space:]]*"error"' "$raw_output_file" \
|
||||
&& grep -Eq "\"kind\"[[:space:]]*:[[:space:]]*\"${expected_kind}\"" "$raw_output_file" \
|
||||
&& [ "$leakage_status" = "not-detected" ] \
|
||||
&& [ "$traceability_status" = "complete" ]; then
|
||||
verdict="pass"
|
||||
fi
|
||||
|
||||
cat >"$markdown_file" <<EOF
|
||||
### ${scenario_id}
|
||||
|
||||
- Date (UTC): \`${utc_now}\`
|
||||
- Operator: \`${operator}\`
|
||||
- Host: \`${host_name}\`
|
||||
- Execution Path: \`${execution_path}\`
|
||||
- Binary Path: \`${binary_label}\`
|
||||
- Binary Version: \`${binary_version}\`
|
||||
- Commit SHA: \`${commit_sha}\`
|
||||
- Branch: \`${branch_name}\`
|
||||
- Driver: \`${driver}\`
|
||||
- Scenario ID: \`${scenario_id}\`
|
||||
- Command: \`${command_text}\`
|
||||
- Exit Code: \`${exit_code}\`
|
||||
- Key Output: \`${key_output_summary}\`
|
||||
|
||||
Output (excerpt):
|
||||
\`\`\`
|
||||
${output_excerpt}
|
||||
\`\`\`
|
||||
|
||||
Verdict:
|
||||
- Expected: non-zero exit, \`status=error\`, \`state=error\`, \`error.kind=${expected_kind}\`, no secret leakage
|
||||
- Traceability check: \`${traceability_status}\` (${traceability_summary})
|
||||
- Secret leakage check: \`${leakage_status}\` (${leakage_mode})
|
||||
- Actual: see output excerpt above
|
||||
- Result: \`${verdict}\`
|
||||
EOF
|
||||
|
||||
echo "${scenario_id} -> ${markdown_file} (${verdict})"
|
||||
)
|
||||
}
|
||||
|
||||
if [ "$scope" = "postgres" ] || [ "$scope" = "all" ]; then
|
||||
run_scenario \
|
||||
"PG-CONN-AUTH-001" \
|
||||
"postgres" \
|
||||
"authentication" \
|
||||
"wrong-password" \
|
||||
"exact" \
|
||||
connect --driver postgres --host "$postgres_host" --port "$postgres_port" --database dbtool_demo --username dbtool --password-env DBTOOL_PASSWORD --result-format json
|
||||
|
||||
run_scenario \
|
||||
"PG-CONN-NET-001" \
|
||||
"postgres" \
|
||||
"connection" \
|
||||
"dbtool" \
|
||||
"contextual" \
|
||||
connect --driver postgres --host "$postgres_host" --port "$postgres_net_port" --database dbtool_demo --username dbtool --password-env DBTOOL_PASSWORD --result-format json
|
||||
|
||||
run_scenario \
|
||||
"PG-QUERY-SQL-001" \
|
||||
"postgres" \
|
||||
"query" \
|
||||
"dbtool" \
|
||||
"contextual" \
|
||||
query --driver postgres --host "$postgres_host" --port "$postgres_port" --database dbtool_demo --username dbtool --password-env DBTOOL_PASSWORD --file examples/sql/postgres/failing_query.sql --result-format json
|
||||
fi
|
||||
|
||||
if [ "$scope" = "mysql" ] || [ "$scope" = "all" ]; then
|
||||
run_scenario \
|
||||
"MY-CONN-AUTH-001" \
|
||||
"mysql" \
|
||||
"authentication" \
|
||||
"wrong-password" \
|
||||
"exact" \
|
||||
connect --driver mysql --host "$mysql_host" --port "$mysql_port" --database qa_demo --username dbtool --password-env DBTOOL_PASSWORD --result-format json
|
||||
|
||||
run_scenario \
|
||||
"MY-CONN-NET-001" \
|
||||
"mysql" \
|
||||
"connection" \
|
||||
"dbtool" \
|
||||
"contextual" \
|
||||
connect --driver mysql --host "$mysql_host" --port "$mysql_net_port" --database qa_demo --username dbtool --password-env DBTOOL_PASSWORD --result-format json
|
||||
|
||||
run_scenario \
|
||||
"MY-QUERY-SQL-001" \
|
||||
"mysql" \
|
||||
"query" \
|
||||
"dbtool" \
|
||||
"contextual" \
|
||||
query --driver mysql --host "$mysql_host" --port "$mysql_port" --database qa_demo --username dbtool --password-env DBTOOL_PASSWORD --file examples/sql/mysql/failing_query.sql --result-format json
|
||||
fi
|
||||
|
||||
echo "evidence written to: $output_dir"
|
||||
Reference in New Issue
Block a user