antipode.info/zabbix-php-fpm
3
0
Fork 0
mirror of https://github.com/rvalitov/zabbix-php-fpm.git synced 2023-11-05 03:30:27 +01:00
Code Issues Releases Wiki Activity

Fix discovery bugs: pools duplicate names and IP TCP connections (#38)

Browse Source
This commit is contained in:
Ramil Valitov 2020-07-07 23:38:01 +03:00 committed by GitHub
parent e110535232
commit c129bf941f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 164 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
tests/all.sh
View File

@ -3,44 +3,83 @@
#https://github.com/rvalitov/zabbix-php-fpm #https://github.com/rvalitov/zabbix-php-fpm
#This script is used for testing #This script is used for testing
MAX_POOLS=3
MAX_PORTS=3
MIN_PORT=9000
copyPool() {
ORIGINAL_FILE=$1
POOL_NAME=$2
POOL_SOCKET=$3
POOL_TYPE=$4
POOL_DIR=$(dirname "${ORIGINAL_FILE}")
PHP_VERSION=$(echo "$POOL_DIR" | grep -oP "(\d\.\d)")
NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf"
sudo cp "$ORIGINAL_FILE" "$NEW_POOL_FILE"
#Add status path
sudo sed -i 's#;pm.status_path.*#pm.status_path = /php-fpm-status#' "$NEW_POOL_FILE"
#Set pool manager
sudo sed -i "s#pm = dynamic#pm = $POOL_TYPE#" "$NEW_POOL_FILE"
#Socket
sudo sed -i "s#listen =.*#listen = $POOL_SOCKET#" "$NEW_POOL_FILE"
#Pool name
sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE"
}
setupPool() { setupPool() {
POOL_FILE=$1 POOL_FILE=$1
POOL_DIR=$(dirname "${POOL_FILE}") POOL_DIR=$(dirname "${POOL_FILE}")
PHP_VERSION=$(echo "$POOL_DIR" | grep -oP "(\d\.\d)") PHP_VERSION=$(echo "$POOL_DIR" | grep -oP "(\d\.\d)")
#Delete all active pools except www.conf:
find "$POOL_DIR" -name '*.conf' -type f -not -name 'www.conf' -exec rm -rf {} \;
#Add status path #Add status path
echo 'pm.status_path = /php-fpm-status' | sudo tee -a "$POOL_FILE" sudo sed -i 's#;pm.status_path.*#pm.status_path = /php-fpm-status#' "$POOL_FILE"
#Set pool manager #Set pool manager
sudo sed -i 's#pm = dynamic#pm = static#' "$POOL_FILE" sudo sed -i 's#pm = dynamic#pm = static#' "$POOL_FILE"
#Make copies and create new socket pools #Create new socket pools
MAX_POOLS=50
for ((c = 1; c <= MAX_POOLS; c++)); do for ((c = 1; c <= MAX_POOLS; c++)); do
POOL_NAME="socket$c" POOL_NAME="socket$c"
NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf" POOL_SOCKET="/run/php/php${PHP_VERSION}-fpm-${POOL_NAME}.sock"
sudo cp "$POOL_FILE" "$NEW_POOL_FILE" copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_SOCKET" "static"
sudo sed -i "s#listen =.*#listen = /run/php/php${PHP_VERSION}-fpm-${POOL_NAME}.sock#" "$NEW_POOL_FILE"
sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE"
done done
#Make copies and create HTTP pools #Create TCP port based pools
MAX_PORTS=50
#Division on 1 is required to convert from float to integer #Division on 1 is required to convert from float to integer
START_PORT=$(echo "(9000 + $PHP_VERSION * 100)/1" | bc) START_PORT=$(echo "($MIN_PORT + $PHP_VERSION * 100 + 1)/1" | bc)
for ((c = 1; c <= MAX_PORTS; c++)); do for ((c = 1; c <= MAX_PORTS; c++)); do
POOL_NAME="http$c" POOL_NAME="port$c"
POOL_PORT=$(echo "($START_PORT + $c)/1" | bc) POOL_PORT=$(echo "($START_PORT + $c)/1" | bc)
NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf" copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_PORT" "static"
sudo cp "$POOL_FILE" "$NEW_POOL_FILE"
sudo sed -i "s#listen =.*#listen = 127.0.0.1:$POOL_PORT#" "$NEW_POOL_FILE"
sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE"
done done
#Create TCP IPv4 localhost pool
POOL_NAME="localhost"
POOL_PORT=$(echo "($MIN_PORT + $PHP_VERSION * 100)/1" | bc)
POOL_SOCKET="127.0.0.1:$POOL_PORT"
copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_SOCKET" "static"
sudo service "php${PHP_VERSION}-fpm" restart sudo service "php${PHP_VERSION}-fpm" restart
} }
setupPools() {
PHP_LIST=$(find /etc/php/ -name 'www.conf' -type f)
while IFS= read -r pool; do
if [[ -n $pool ]]; then
setupPool "$pool"
fi
done <<<"$PHP_LIST"
}
getNumberOfPHPVersions() {
PHP_COUNT=$(find /etc/php/ -name 'www.conf' -type f | wc -l)
echo "$PHP_COUNT"
}
getAnySocket() { getAnySocket() {
#Get any socket of PHP-FPM: #Get any socket of PHP-FPM:
PHP_FIRST=$(find /etc/php/ -name 'www.conf' -type f | head -n1) PHP_FIRST=$(find /etc/php/ -name 'www.conf' -type f | head -n1)
@ -60,6 +99,10 @@ getAnyPort() {
oneTimeSetUp() { oneTimeSetUp() {
echo "Started job $TRAVIS_JOB_NAME" echo "Started job $TRAVIS_JOB_NAME"
echo "Host info:"
nslookup localhost
sudo ifconfig
sudo cat /etc/hosts
echo "Copying Zabbix files..." echo "Copying Zabbix files..."
#Install files: #Install files:
sudo cp "$TRAVIS_BUILD_DIR/zabbix/zabbix_php_fpm_discovery.sh" "/etc/zabbix" sudo cp "$TRAVIS_BUILD_DIR/zabbix/zabbix_php_fpm_discovery.sh" "/etc/zabbix"
@ -75,12 +118,7 @@ oneTimeSetUp() {
echo "Setup PHP-FPM..." echo "Setup PHP-FPM..."
#Setup PHP-FPM pools: #Setup PHP-FPM pools:
PHP_LIST=$(find /etc/php/ -name 'www.conf' -type f) setupPools
while IFS= read -r pool; do
if [[ -n $pool ]]; then
setupPool "$pool"
fi
done <<<"$PHP_LIST"
echo "All done, starting tests..." echo "All done, starting tests..."
} }
@ -116,6 +154,7 @@ testStatusScriptSocket() {
DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status") DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status")
IS_OK=$(echo "$DATA" | grep -F '{"pool":"') IS_OK=$(echo "$DATA" | grep -F '{"pool":"')
assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK"
echo "Success test of $PHP_POOL"
} }
testStatusScriptPort() { testStatusScriptPort() {
@ -126,6 +165,7 @@ testStatusScriptPort() {
DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status") DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status")
IS_OK=$(echo "$DATA" | grep -F '{"pool":"') IS_OK=$(echo "$DATA" | grep -F '{"pool":"')
assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK"
echo "Success test of $PHP_POOL"
} }
testZabbixStatusSocket() { testZabbixStatusSocket() {
@ -134,6 +174,7 @@ testZabbixStatusSocket() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"]) DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"])
IS_OK=$(echo "$DATA" | grep -F '{"pool":"') IS_OK=$(echo "$DATA" | grep -F '{"pool":"')
assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK"
echo "Success test of $PHP_POOL"
} }
testZabbixStatusPort() { testZabbixStatusPort() {
@ -143,6 +184,7 @@ testZabbixStatusPort() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"]) DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"])
IS_OK=$(echo "$DATA" | grep -F '{"pool":"') IS_OK=$(echo "$DATA" | grep -F '{"pool":"')
assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK"
echo "Success test of $PHP_POOL"
} }
testDiscoverScriptReturnsData() { testDiscoverScriptReturnsData() {
@ -151,5 +193,38 @@ testDiscoverScriptReturnsData() {
assertNotNull "Discover script failed: $DATA" "$IS_OK" assertNotNull "Discover script failed: $DATA" "$IS_OK"
} }
testDiscoverScriptDebug() {
DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_discovery.sh" "debug" "/php-fpm-status")
ERRORS_LIST=$(echo "$DATA" | grep -F 'Error:')
assertNull "Discover script errors: $DATA" "$ERRORS_LIST"
}
testZabbixDiscoverReturnsData() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"])
IS_OK=$(echo "$DATA" | grep -F '{"data":[{"{#POOLNAME}"')
assertNotNull "Discover script failed: $DATA" "$IS_OK"
}
testZabbixDiscoverNumberOfSocketPools() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"])
NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"socket1",' | wc -l)
PHP_COUNT=$(getNumberOfPHPVersions)
assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS"
}
testZabbixDiscoverNumberOfIPPools() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"])
NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"localhost",' | wc -l)
PHP_COUNT=$(getNumberOfPHPVersions)
assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS"
}
testZabbixDiscoverNumberOfPortPools() {
DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"])
NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"port1",' | wc -l)
PHP_COUNT=$(getNumberOfPHPVersions)
assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS"
}
# Load shUnit2. # Load shUnit2.
. shunit2 . shunit2

118
zabbix/zabbix_php_fpm_discovery.sh
View File

@ -7,6 +7,7 @@ S_PS=$(type -P ps)
S_GREP=$(type -P grep) S_GREP=$(type -P grep)
S_AWK=$(type -P awk) S_AWK=$(type -P awk)
S_SORT=$(type -P sort) S_SORT=$(type -P sort)
S_UNIQ=$(type -P uniq)
S_HEAD=$(type -P head) S_HEAD=$(type -P head)
S_LSOF=$(type -P lsof) S_LSOF=$(type -P lsof)
S_JQ=$(type -P jq) S_JQ=$(type -P jq)
@ -33,6 +34,10 @@ if [[ ! -f $S_SORT ]]; then
${S_ECHO} "Utility 'sort' not found. Please, install it first." ${S_ECHO} "Utility 'sort' not found. Please, install it first."
exit 1 exit 1
fi fi
if [[ ! -f $S_UNIQ ]]; then
${S_ECHO} "Utility 'uniq' not found. Please, install it first."
exit 1
fi
if [[ ! -f $S_HEAD ]]; then if [[ ! -f $S_HEAD ]]; then
${S_ECHO} "Utility 'head' not found. Please, install it first." ${S_ECHO} "Utility 'head' not found. Please, install it first."
exit 1 exit 1
@ -133,6 +138,7 @@ function ProcessPool() {
POOL_NAME=$1 POOL_NAME=$1
POOL_SOCKET=$2 POOL_SOCKET=$2
if [[ -z ${POOL_NAME} ]] || [[ -z ${POOL_SOCKET} ]]; then if [[ -z ${POOL_NAME} ]] || [[ -z ${POOL_SOCKET} ]]; then
PrintDebug "Invalid arguments for ProcessPool"
return 0 return 0
fi fi
@ -215,14 +221,21 @@ fi
mapfile -t PS_LIST < <($S_PS ax | $S_GREP -F "php-fpm: pool " | $S_GREP -F -v "grep") mapfile -t PS_LIST < <($S_PS ax | $S_GREP -F "php-fpm: pool " | $S_GREP -F -v "grep")
# shellcheck disable=SC2016 # shellcheck disable=SC2016
POOL_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_AWK '{print $NF}' | $S_SORT -u) POOL_NAMES_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_AWK '{print $NF}' | $S_SORT -u)
POOL_FIRST=0 POOL_FIRST=0
#We store the resulting JSON data for Zabbix in the following var: #We store the resulting JSON data for Zabbix in the following var:
RESULT_DATA="{\"data\":[" RESULT_DATA="{\"data\":["
while IFS= read -r line; do while IFS= read -r line; do
# shellcheck disable=SC2016 # shellcheck disable=SC2016
POOL_PID=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_GREP -F -w "php-fpm: pool $line" | $S_HEAD -1 | $S_AWK '{print $1}') POOL_PID_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_GREP -F -w "php-fpm: pool $line" | $S_AWK '{print $1}')
if [[ -n $POOL_PID ]]; then POOL_PID_ARGS=""
while IFS= read -r POOL_PID; do
if [[ -n $POOL_PID ]]; then
POOL_PID_ARGS="$POOL_PID_ARGS -p $POOL_PID"
fi
done <<<"$POOL_PID_LIST"
if [[ -n $POOL_PID_ARGS ]]; then
#We search for socket or IP address and port #We search for socket or IP address and port
#Socket example: #Socket example:
#php-fpm7. 25897 root 9u unix 0x000000006509e31f 0t0 58381847 /run/php/php7.3-fpm.sock type=STREAM #php-fpm7. 25897 root 9u unix 0x000000006509e31f 0t0 58381847 /run/php/php7.3-fpm.sock type=STREAM
@ -235,84 +248,85 @@ while IFS= read -r line; do
#php-fpm7. 1203 www-data 8u IPv4 15070917 0t0 TCP localhost.localdomain:23054->localhost.localdomain:postgresql (ESTABLISHED) #php-fpm7. 1203 www-data 8u IPv4 15070917 0t0 TCP localhost.localdomain:23054->localhost.localdomain:postgresql (ESTABLISHED)
#More info at https://github.com/rvalitov/zabbix-php-fpm/issues/12 #More info at https://github.com/rvalitov/zabbix-php-fpm/issues/12
PrintDebug "Started analysis of pool $line, PID $POOL_PID" PrintDebug "Started analysis of pool $line, PID(s): $POOL_PID_ARGS"
#Extract only important information: #Extract only important information:
#Use -P to show port number instead of port name, see https://github.com/rvalitov/zabbix-php-fpm/issues/24 #Use -P to show port number instead of port name, see https://github.com/rvalitov/zabbix-php-fpm/issues/24
POOL_PARAMS_LIST=$($S_LSOF -P -p "$POOL_PID" 2>/dev/null | $S_GREP -w -e "unix" -e "TCP") #Use -n flag to show IP address and not convert it to domain name (like localhost)
#Sometimes different PHP-FPM versions may have the same names of pools, so we need to consider that.
# It's considered that a pair of pool name and socket must be unique.
#Sorting is required, because uniq needs it
POOL_PARAMS_LIST=$($S_LSOF -n -P $POOL_PID_ARGS 2>/dev/null | $S_GREP -w -e "unix" -e "TCP" | $S_SORT -u | $S_UNIQ -f8)
FOUND_POOL="" FOUND_POOL=""
while IFS= read -r pool; do while IFS= read -r pool; do
if [[ -n $pool ]]; then if [[ -n $pool ]]; then
if [[ -z $FOUND_POOL ]]; then PrintDebug "Checking process: $pool"
PrintDebug "Checking process: $pool" # shellcheck disable=SC2016
# shellcheck disable=SC2016 POOL_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $5}')
POOL_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $5}') # shellcheck disable=SC2016
# shellcheck disable=SC2016 POOL_SOCKET=$(${S_ECHO} "${pool}" | $S_AWK '{print $9}')
POOL_SOCKET=$(${S_ECHO} "${pool}" | $S_AWK '{print $9}') if [[ -n $POOL_TYPE ]] && [[ -n $POOL_SOCKET ]]; then
if [[ -n $POOL_TYPE ]] && [[ -n $POOL_SOCKET ]]; then if [[ $POOL_TYPE == "unix" ]]; then
if [[ $POOL_TYPE == "unix" ]]; then #We have a socket here, test if it's actually a socket:
#We have a socket here, test if it's actually a socket: if [[ -S $POOL_SOCKET ]]; then
if [[ -S $POOL_SOCKET ]]; then PrintDebug "Found socket $POOL_SOCKET"
PrintDebug "Found socket $POOL_SOCKET" ProcessPool "${line}" "${POOL_SOCKET}"
POOL_STATUS=$?
if [[ ${POOL_STATUS} -gt 0 ]]; then
FOUND_POOL="1"
PrintDebug "Success: socket $POOL_SOCKET returned valid status data"
EncodeToJson "${line}" "${POOL_SOCKET}"
else
PrintDebug "Error: socket $POOL_SOCKET didn't return valid data"
fi
else
PrintDebug "Error: specified socket $POOL_SOCKET is not valid"
fi
elif [[ $POOL_TYPE == "IPv4" ]] || [[ $POOL_TYPE == "IPv6" ]]; then
#We have a TCP connection here, check it:
# shellcheck disable=SC2016
CONNECTION_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $8}')
if [[ $CONNECTION_TYPE == "TCP" ]]; then
#The connection must have state LISTEN:
LISTEN=$(${S_ECHO} "${pool}" | $S_GREP -F -w "(LISTEN)")
if [[ -n $LISTEN ]]; then
#Check and replace * to localhost if it's found. Asterisk means that the PHP listens on
#all interfaces.
PrintDebug "Found TCP connection $POOL_SOCKET"
POOL_SOCKET=${POOL_SOCKET/\*:/localhost:}
PrintDebug "Processed TCP connection $POOL_SOCKET"
ProcessPool "${line}" "${POOL_SOCKET}" ProcessPool "${line}" "${POOL_SOCKET}"
POOL_STATUS=$? POOL_STATUS=$?
if [[ ${POOL_STATUS} -gt 0 ]]; then if [[ ${POOL_STATUS} -gt 0 ]]; then
FOUND_POOL="1" FOUND_POOL="1"
PrintDebug "Success: socket $POOL_SOCKET returned valid status data" PrintDebug "Success: TCP connection $POOL_SOCKET returned valid status data"
EncodeToJson "${line}" "${POOL_SOCKET}"
else else
PrintDebug "Error: socket $POOL_SOCKET didn't return valid data" PrintDebug "Error: TCP connection $POOL_SOCKET didn't return valid data"
fi fi
else else
PrintDebug "Error: specified socket $POOL_SOCKET is not valid" PrintDebug "Warning: expected connection state must be LISTEN, but it was not detected"
fi
elif [[ $POOL_TYPE == "IPv4" ]] || [[ $POOL_TYPE == "IPv6" ]]; then
#We have a TCP connection here, check it:
# shellcheck disable=SC2016
CONNECTION_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $8}')
if [[ $CONNECTION_TYPE == "TCP" ]]; then
#The connection must have state LISTEN:
LISTEN=$(${S_ECHO} "${pool}" | $S_GREP -F -w "(LISTEN)")
if [[ -n $LISTEN ]]; then
#Check and replace * to localhost if it's found. Asterisk means that the PHP listens on
#all interfaces.
POOL_SOCKET=$(${S_ECHO} -n "${POOL_SOCKET/*:/localhost:}")
PrintDebug "Found TCP connection $POOL_SOCKET"
ProcessPool "${line}" "${POOL_SOCKET}"
POOL_STATUS=$?
if [[ ${POOL_STATUS} -gt 0 ]]; then
FOUND_POOL="1"
PrintDebug "Success: TCP connection $POOL_SOCKET returned valid status data"
else
PrintDebug "Error: TCP connection $POOL_SOCKET didn't return valid data"
fi
else
PrintDebug "Warning: expected connection state must be LISTEN, but it was not detected"
fi
else
PrintDebug "Warning: expected connection type is TCP, but found $CONNECTION_TYPE"
fi fi
else else
PrintDebug "Unsupported type $POOL_TYPE, skipping" PrintDebug "Warning: expected connection type is TCP, but found $CONNECTION_TYPE"
fi fi
else else
PrintDebug "Warning: pool type or socket is empty" PrintDebug "Unsupported type $POOL_TYPE, skipping"
fi fi
else else
PrintDebug "Pool already found, skipping process: $pool" PrintDebug "Warning: pool type or socket is empty"
fi fi
else else
PrintDebug "Error: failed to get process information. Probably insufficient privileges. Use sudo or run this script under root." PrintDebug "Error: failed to get process information. Probably insufficient privileges. Use sudo or run this script under root."
fi fi
done <<<"$POOL_PARAMS_LIST" done <<<"$POOL_PARAMS_LIST"
if [[ -n ${FOUND_POOL} ]]; then if [[ -z ${FOUND_POOL} ]]; then
EncodeToJson "${line}" "${POOL_SOCKET}"
else
PrintDebug "Error: failed to discover information for pool $line" PrintDebug "Error: failed to discover information for pool $line"
fi fi
else else
PrintDebug "Error: failed to find PID for pool $line" PrintDebug "Error: failed to find PID for pool $line"
fi fi
done <<<"$POOL_LIST" done <<<"$POOL_NAMES_LIST"
PrintDebug "Processing pools from old cache..." PrintDebug "Processing pools from old cache..."
for CACHE_ITEM in "${CACHE[@]}"; do for CACHE_ITEM in "${CACHE[@]}"; do