#!/bin/bash
#
# Debug utility: packages log files, and relevant debug information
#                needed for full diagnosis of Spacewalk Server issues.
#
# Copyright (c) 2008--2018 Red Hat, Inc.
# Copyright (c) 2025 SUSE LLC
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.
#


if [ "$(id -u)" != "0" ] ; then
  echo "This script must be run as root."
  exit
fi

BASE_DIR=/tmp
IS_SUSE=0
NO_REPORTS=0
NO_COMPRESSION=0
MAX_LOG_AGE=10
if grep -iq '^ID_LIKE=.*suse' /etc/os-release; then
    IS_SUSE=1
fi
if [ "$SPACEWALK_DEBUG_NO_REPORTS" = 1 ]; then
    NO_REPORTS=1
fi

usage() {
    echo "Usage:"
    echo "$0 [OPTIONS]"
    echo "Debug utility that packages log files and other information"
    echo
    echo "  OPTIONS:"
    echo "    --help                Display usage and exit"
    echo "    --dir                 Destination directory ($BASE_DIR)"
    echo "    --log-age             Set the maximum age (in days) for extracted logs ($MAX_LOG_AGE)"
    echo "    --no-reports          Do not run spacewalk-report. Useful if spacewalk-debug takes too long"
    echo "    --no-compression      Do not compress spacewalk-debug destination folder"
    exit 1
}

while [ ${#} -ne 0 ]; do
    arg="$1"
    case "$arg" in
        --help)
            usage 0
            ;;
        --dir)
            shift
            BASE_DIR=$1
            ;;
        --log-age)
            shift
            MAX_LOG_AGE=$1
            ;;
        --no-reports)
            NO_REPORTS=1
            ;;
        --no-compression)
            NO_COMPRESSION=1
            ;;
        --*)
            echo "Unknown option $arg (use --help)"
            exit 1
            ;;
        *)
            echo "Too many arguments (use --help)"
            exit 1
            ;;
    esac
    shift
done

if [ ! -d "$BASE_DIR" ]; then
    mkdir -p "$BASE_DIR" || (echo "Unable to create directory $BASE_DIR"; exit 1)
fi

# clean any previous run
rm -rf "$BASE_DIR/spacewalk-debug"/*

# Make sure BASE_DIR is not relative
BASE_DIR=$(cd "$BASE_DIR" && pwd)

DIR="$BASE_DIR/spacewalk-debug"
TARBALL="$BASE_DIR/spacewalk-debug.tar.bz2"

mkdir -p "$DIR" || (echo "Unable to create directory $DIR"; exit 1)
chmod 700 "$DIR"
cd "$DIR" || (echo "Unable to change to the output directory"; exit 1)

echo "Collecting and packaging relevant diagnostic information."
if [ $NO_REPORTS -eq 1 ] ; then
    echo "NOTE: Running with '--no-reports' means 'spacewalk-report' won't be executed."
fi
echo "Warning: this may take some time..."

mkdir -p "$DIR/conf/httpd"
mkdir -p "$DIR/conf/tomcat"
mkdir -p "$DIR/conf/cobbler"
mkdir -p "$DIR/conf/rhn/sysconfig"
mkdir -p "$DIR/database"
mkdir -p "$DIR/httpd-logs"
mkdir -p "$DIR/tomcat-logs"
mkdir -p "$DIR/cobbler-logs"
mkdir -p "$DIR/rhn-logs/rhn"
mkdir -p "$DIR/config-defaults"
mkdir -p "$DIR/kickstarts"
mkdir -p "$DIR/cobbler-lib"
mkdir -p "$DIR/tasko"
mkdir -p "$DIR/salt-states"
if [ -f /usr/bin/journalctl ]; then
  mkdir -p "$DIR/systemd"
fi

echo "    * copying configuration information"
if [ -d /etc/httpd ]; then
    cp -fapRd /etc/httpd/conf* "$DIR/conf/httpd"
elif [ -d /etc/apache2 ]; then
    cp -fapRd /etc/apache2/conf* "$DIR/conf/httpd"
    cp -fapRd /etc/apache2/vhosts* "$DIR/conf/httpd"
fi
cp -fapRd /etc/rhn "$DIR/conf/rhn"
cp -fapRd /etc/sysconfig/rhn "$DIR/conf/rhn/sysconfig"

# there might be backups of rhn.conf so clean them up as well (bsc#1146419)
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^server.susemanager.mirrcred_pass.*/server.susemanager.mirrcred_pass = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^server.secret_key.*/server.secret_key = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^session_secret_.*/session_secret_N = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^web.session_swap_secret_.*/web.session_swap_secret_N = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^db_password.*/db_password = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^report_db_password.*/report_db_password = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'rhn.conf*' -exec sed -i 's/^server.satellite.http_proxy_password.*/server.satellite.http_proxy_password = <replaced_by_debug_tool>/' {} \;

# cleanup also some other files
find "$DIR/conf/rhn" -name 'uln.conf*' -exec sed -i 's/^password.*/password = <replaced_by_debug_tool>/' {} \;
find "$DIR/conf/rhn" -name 'signing.conf*' -exec sed -i 's/^GPGPASS=.*/GPGPASS=<replaced_by_debug_tool>/' {} \;

if [ -f /etc/tnsnames.ora ] ; then
    cp -fad /etc/tnsnames.ora "$DIR/conf"
fi

# Copy generated and existing SLS files
echo "    * copying Salt state files"
D_IFS=$IFS
IFS=","
# shellcheck disable=SC2140
for sls_dir in "/usr/share/susemanager/salt/","static-sls","static SLS" \
               "/srv/susemanager/salt/","generated-sls","generated SLS"; do
    # shellcheck disable=SC2086
    set $sls_dir
    if [ -d "$1" ] && [ "$(ls -A "$1")" ]; then
	    echo -e "    \t- copying $3"
	    cp -fa "$1" "$DIR/salt-states/$2"
    fi
done
IFS=$D_IFS

mkdir -p "$DIR/salt-states/custom-sls/"
while read -r sls; do
    echo -e "    \t- copying custom SLS"
    if [[ $(file "$sls") = *"text"* ]];  then
	    cp --parents -fa "$sls" "$DIR/salt-states/custom-sls/"
    else
      echo "$sls" >> "$DIR/salt-states/custom-sls/other_files"
    fi
done < <(find /srv/salt/ -type f)

MATCHER_DATA_DIR=/var/lib/spacewalk/subscription-matcher
if [ -d $MATCHER_DATA_DIR ]; then
  echo "    * copying Subscription Matcher files"
  cp -fa $MATCHER_DATA_DIR "$DIR/"
fi

echo "    * copying logs"
if [ -f /usr/bin/journalctl ]; then
  /usr/bin/journalctl -m > "$DIR/systemd/journalctl.log"
fi

if [ -d /var/log/httpd ]; then
    cp -fapRd /var/log/httpd "$DIR/httpd-logs"
elif [ -d /var/log/apache2 ]; then
    cp -fapRd /var/log/apache2 "$DIR/httpd-logs"
fi

# copy rhn logs
if [ -d /var/log/rhn ]; then
    cp -fapd  /var/log/rhn/*.log* "$DIR/rhn-logs/rhn"

    # check for reposync dir
    if [ -d /var/log/rhn/reposync ]; then
        cp -fapRd /var/log/rhn/reposync "$DIR/rhn-logs/rhn"
    fi

    # check for search dir
    if [ -d /var/log/rhn/search ]; then
        cp -fapRd /var/log/rhn/search "$DIR/rhn-logs/rhn"
    fi

    # check mgr-create-bootstrap-repo dir
    if [ -d /var/log/rhn/mgr-create-bootstrap-repo ]; then
        cp -fapRd /var/log/rhn/mgr-create-bootstrap-repo "$DIR/rhn-logs/rhn"
    fi
fi

if [ -e /var/log/susemanager ]; then
    cp -fa /var/log/susemanager* "$DIR/rhn-logs"
fi

if [ -d /var/log/salt ]; then
    mkdir -p "$DIR/salt-logs"
    cp -fa /var/log/salt "$DIR/salt-logs"
fi

# tomcat for spacewalk 400+
for tomcat in tomcat tomcat5 tomcat6 ; do
  if [ -d /etc/$tomcat ]; then
    cp -fa /etc/$tomcat "$DIR/conf/tomcat"
  fi
  if [ -d /var/log/$tomcat ]; then
    cp -fa /var/log/$tomcat "$DIR/tomcat-logs"
  fi
done

# copying the /usr/share/rhn/config-defaults
echo "    * copying config-defaults files"
if [ -d /usr/share/rhn/config-defaults ]; then
    cp -fa /usr/share/rhn/config-defaults/* "$DIR/config-defaults"
fi

# CLM - Save Current Environment Version Details
echo "    * querying CLM project history"
mkdir -p "$DIR/clm/projects"
echo "SELECT id, label, first_env_id FROM susecontentproject;" | /usr/bin/spacewalk-sql --select-mode - | tail -n +3 | head -n -2 > "$DIR/clm/tmp_projects.txt"

while read -r project; do
  project_id=$(echo "$project" | awk '{print $1}' |  tr -d '[:space:]')
  project_label=$(echo "$project" | awk '{print $3}' | tr -d '[:space:]')
  first_env_id=$(echo "$project" | awk  '{print $5}' | tr -d '[:space:]')

  curr_env=$first_env_id
  while [ -n "$curr_env" ]; do
    env_label=$(echo "SELECT label FROM susecontentenvironment WHERE id=$curr_env;" | /usr/bin/spacewalk-sql --select-mode - | tail -n +3 | head -n -2 | tr -d '[:space:]')
    env_version=$(echo "SELECT version FROM susecontentenvironment WHERE id=$curr_env;" | /usr/bin/spacewalk-sql --select-mode - | tail -n +3 | head -n -2 | tr -d '[:space:]')
    # shellcheck disable=SC2129
    echo "## Environment: $env_label" >> "$DIR/clm/projects/$project_label.txt"
    echo "SELECT message, version, created FROM susecontentprojecthistoryentry WHERE project_id=$project_id AND version=$env_version ;" | /usr/bin/spacewalk-sql --select-mode - | head -n -2 >> "$DIR/clm/projects/$project_label.txt"
    echo -e "\n\n" >> "$DIR/clm/projects/$project_label.txt"
    next_env=$(echo "SELECT next_env_id FROM susecontentenvironment WHERE id=$curr_env;" | /usr/bin/spacewalk-sql --select-mode - | tail -n +3 | head -n -2 | awk '{print $1}' | tr -d '[:space:]')
    curr_env="$next_env"
  done
done < "$DIR/clm/tmp_projects.txt"

rm "$DIR/clm/tmp_projects.txt"

# Save all CLM releated DB Tables to clm/full_tables.txt
echo "    * querying CLM tables"
for i in $(echo "\\dt;" | /usr/bin/spacewalk-sql --select-mode - |grep susecontent | awk '{print $3}'); do echo "$i"; echo "SELECT * FROM $i;" | /usr/bin/spacewalk-sql --select-mode -; done > "$DIR/clm/full_tables.txt"

#cobbler stuff
echo "    * copying cobbler files"
if [ -d /etc/cobbler ]; then
    cp -fa /etc/cobbler/* "$DIR/conf/cobbler"
fi
if [ -d /var/log/cobbler ]; then
   cp -fa /var/log/cobbler/* "$DIR/cobbler-logs"
fi
if [ -d /var/lib/cobbler ]; then
   cp -fa /var/lib/cobbler/snippets "$DIR/cobbler-lib/"
   cp -fa /var/lib/cobbler/config "$DIR/cobbler-lib/" 2>/dev/null
   cp -fa /var/lib/cobbler/kickstarts "$DIR/cobbler-lib/"
   cp -fa /var/lib/cobbler/triggers "$DIR/cobbler-lib/"
   cp -fa /var/lib/cobbler/collections "$DIR/cobbler-lib/"
fi
if [ -d /var/lib/rhn/kickstarts ]; then
   cp -fa /var/lib/rhn/kickstarts/* "$DIR/kickstarts/"
fi

# Remove passwords from cobbler settings
find "$DIR/conf/cobbler" -type f -name 'settings*' -exec sed -i 's/^default_password_crypted.*/default_password_crypted <replaced_by_debug_tool>/' {} \;

# ssl-build
if [ -d /root/ssl-build ] ; then
    echo "    * copying ssl-build"
    mkdir -p "$DIR/ssl-build"
    # NOTE: cp -a == cp -pRd
    ls -lR /root/ssl-build/ > "$DIR/ssl-build/ls-lR.txt" 2> /dev/null
fi

# /etc/sudoers
if [ -f /etc/sudoers ] || [ -d /etc/sudoers.d ] ; then
	echo "    * copying /etc/sudoers*"
	cp -fa /etc/sudoers* "$DIR/conf"
fi

# /etc/passwd
if [ -f /etc/passwd ] ; then
	echo "    * copying apache, postgres, tomcat entries from /etc/passwd"
	getent passwd apache wwwrun tomcat postgres > "$DIR/conf/passwd"
fi

# /etc/group
if [ -f /etc/group ] ; then
	echo "    * copying apache, postgres, tomcat entries from /etc/group"
	getent group apache www tomcat postgres > "$DIR/conf/group"
fi

echo "    * querying RPM database"
rpm -qa --last > "$DIR/rpm-manifest"
rpm -qa | sort > "$DIR/rpm-manifest-clean"

echo "    * querying schema version, database charactersets and database"
/usr/bin/rhn-schema-version > "$DIR/database-schema-version"
/usr/bin/rhn-charsets > "$DIR/database-character-sets"

if [ -f /usr/bin/spacewalk-sql ] ; then
    USERS_TZ_LC_SQL="""
        SELECT w.login as login,
               tz.display_name as timezone,
               ui.preferred_locale as locale
          FROM web_contact w
               inner join rhnuserinfo ui on ui.user_id = w.id
               left outer join rhntimezone tz on ui.timezone_id = tz.id
        ORDER BY login;
    """
    echo "    * querying users timezone and localization preferences"
    echo "$USERS_TZ_LC_SQL" | /usr/bin/spacewalk-sql --select-mode - > "$DIR/users-preferences"
fi

echo "    * get diskspace available"
df -h > "$DIR/diskinfo"

echo "    * get database settings"
QUERY="SHOW ALL;"
echo "$QUERY" | /usr/bin/spacewalk-sql --select-mode - > "$DIR/database/db-settings.log"

echo "    * get schema statistics"
/usr/bin/rhn-schema-stats "$DIR/database/schema-stats.log"

echo "    * get tables statistics"
QUERY="SELECT * FROM pg_stat_user_tables;"
echo "$QUERY" | /usr/bin/spacewalk-sql --select-mode - > "${DIR}/database/table_stats.log"

echo "    * get per table configuration options"
QUERY="SELECT relname, reloptions FROM pg_class JOIN pg_namespace ON  pg_namespace.oid = pg_class.relnamespace WHERE pg_namespace.nspname = 'public' AND pg_class.relkind = 'r';"
echo "$QUERY" | /usr/bin/spacewalk-sql --select-mode - > "${DIR}/database/table_options.log"

if [ -d /var/log/spacewalk/schema-upgrade ] ; then
	echo "    * copying schema upgrade logs"
	mkdir -p "$DIR/schema-upgrade-logs"
	cp -pr /var/log/spacewalk/schema-upgrade/* "$DIR/schema-upgrade-logs"
fi

echo "    * copying Postgresql procedures information"
QUERY="SELECT proname, prosrc FROM pg_catalog.pg_proc;"
echo "$QUERY"  | /usr/bin/spacewalk-sql --select-mode - > "$DIR/database/pg_catalog.pg_proc.log"

if [ -f /var/log/audit/audit.log ] ; then
	echo "    * copying audit.log"
	mkdir -p "$DIR/audit-log"
	cp -fa /var/log/audit/audit.log "$DIR/audit-log"
fi

if [ -d /var/log/rhn/tasko/sat ] ; then
	echo "    * copying tasko/sat"
	cp -fa /var/log/rhn/tasko/sat "$DIR/tasko"
fi

if [ $IS_SUSE -eq 1 ]; then
    if [ -d "/var/lib/spacewalk/scc/scc-data" ]; then
        echo  "    * copying SCC data"
        mkdir -p "$DIR/sccdata"
        for i in /var/lib/spacewalk/scc/scc-data/*; do
            cp -fa "$i" "$DIR/sccdata/"
        done
    fi

    if [ -x /usr/bin/spacewalk-report ] && [ $NO_REPORTS -eq 0 ]; then
        # get some inventory info
        echo "    * running spacewalk-report to create reports."
        /usr/bin/spacewalk-report --legacy-report inventory > "$DIR/inventory-report.csv"
        /usr/bin/spacewalk-report --legacy-report channels > "$DIR/channels-report.csv"
    fi

    ls -laR "$(spacewalk-cfg-get documentroot)/pub" > "$DIR/ls-laR-htdocs"
fi

echo "    * timestamping"
echo "Spacewalk debug created on $(date)" > "$DIR/timestamp"

# Skip if MAX_LOG_AGE is not a number
if [ "$MAX_LOG_AGE" -eq "$MAX_LOG_AGE" ]; then
    # Remove all logs older than MAX_LOG_AGE to reduce the size of the tarball
    find "$DIR"/* -regextype sed \( -iregex ".*log\.[0-9]\{1,\}.*" -or -iregex ".*[0-9]\{8\}.*" \
        -or -iregex ".*[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}\(\.\|_[0-9]\{6\}\(_sync\)\?\.\).*" \) \
        -daystart -mtime +"$MAX_LOG_AGE" -not -ipath "*schema*" -delete

        # tasko files need to be parsed separately to find old ones
        for dir in rhn-logs/rhn/tasko tasko/sat; do
            if [ -d "$dir" ]; then
                find "$DIR/$dir" -type f -daystart -mtime +"$MAX_LOG_AGE" -delete
            fi
        done
fi

# task schedules
if [ -f /usr/bin/spacewalk-sql ] ; then
    # Extract task schedules from the database - only the default ones
    TASK_SCHEDULE_DEFAULT_SQL="""
        SELECT id, job_label, bunch_id, org_id,
          active_from, active_till, cron_expr,
          data, created, modified
        FROM rhntaskoschedule
        WHERE cron_expr is not null or job_label like '%default';
    """
    echo "$TASK_SCHEDULE_DEFAULT_SQL" | /usr/bin/spacewalk-sql --select-mode - > "$DIR/tasko/task_schedule_default"

    # Extract task schedules from the database - only the modified ones
    TASK_SCHEDULE_MODIFIED_SQL="""
        SELECT id, job_label, bunch_id, org_id,
          active_from, active_till, cron_expr,
          data, created, modified
        FROM rhntaskoschedule
        WHERE created <> modified;
    """
    echo "$TASK_SCHEDULE_MODIFIED_SQL" | /usr/bin/spacewalk-sql --select-mode - > "$DIR/tasko/task_schedule_modified"
fi

# exclude private keys
find "$DIR" -name "*PRIVATE*" -delete
find "$DIR" -name "server.key*" -delete
find "$DIR" -name "server.pem*" -delete
find "$DIR" -name "rhn-org-httpd-ssl*" -delete

# Remove bootloader linux+initrd files
if [ -d "$DIR/salt-states/generated-sls/bootloader/" ]; then
    find "$DIR/salt-states/generated-sls/bootloader/" -type f -exec truncate -s 0 {} \;
    touch "$DIR/salt-states/generated-sls/bootloader/DATA_GOT_TRUNCATED_VIA_SUPPORTCONFIG"
fi

# fix permissions
chmod -R 700 "$DIR"

if [ $NO_COMPRESSION -eq 1 ] ; then
  echo
  echo "Debug dump created, stored in $DIR"
else
  echo "    * creating tarball (may take some time): $TARBALL"
  # set tarball permissions *before* writing data
  install -m 600 /dev/null "$TARBALL"
  tar -cjf "$TARBALL" \
      -C "$(dirname "$DIR")" \
      "$(basename "$DIR")"

  echo "    * removing temporary debug tree"
  rm -Rf "$DIR"
  echo "Debug dump created, stored in $TARBALL"
fi

