From c129bf941f53df3e34c0eb0abe9a9c464dffa9e8 Mon Sep 17 00:00:00 2001 From: Ramil Valitov Date: Tue, 7 Jul 2020 23:38:01 +0300 Subject: [PATCH] Fix discovery bugs: pools duplicate names and IP TCP connections (#38) --- tests/all.sh | 121 +++++++++++++++++++++++------ zabbix/zabbix_php_fpm_discovery.sh | 118 +++++++++++++++------------- 2 files changed, 164 insertions(+), 75 deletions(-) diff --git a/tests/all.sh b/tests/all.sh index 12bad36..071f4ad 100644 --- a/tests/all.sh +++ b/tests/all.sh @@ -3,44 +3,83 @@ #https://github.com/rvalitov/zabbix-php-fpm #This script is used for testing +MAX_POOLS=3 +MAX_PORTS=3 +MIN_PORT=9000 + +copyPool() { + ORIGINAL_FILE=$1 + POOL_NAME=$2 + POOL_SOCKET=$3 + POOL_TYPE=$4 + POOL_DIR=$(dirname "${ORIGINAL_FILE}") + PHP_VERSION=$(echo "$POOL_DIR" | grep -oP "(\d\.\d)") + + NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf" + sudo cp "$ORIGINAL_FILE" "$NEW_POOL_FILE" + + #Add status path + sudo sed -i 's#;pm.status_path.*#pm.status_path = /php-fpm-status#' "$NEW_POOL_FILE" + #Set pool manager + sudo sed -i "s#pm = dynamic#pm = $POOL_TYPE#" "$NEW_POOL_FILE" + #Socket + sudo sed -i "s#listen =.*#listen = $POOL_SOCKET#" "$NEW_POOL_FILE" + #Pool name + sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE" +} + setupPool() { POOL_FILE=$1 POOL_DIR=$(dirname "${POOL_FILE}") PHP_VERSION=$(echo "$POOL_DIR" | grep -oP "(\d\.\d)") + #Delete all active pools except www.conf: + find "$POOL_DIR" -name '*.conf' -type f -not -name 'www.conf' -exec rm -rf {} \; + #Add status path - echo 'pm.status_path = /php-fpm-status' | sudo tee -a "$POOL_FILE" + sudo sed -i 's#;pm.status_path.*#pm.status_path = /php-fpm-status#' "$POOL_FILE" #Set pool manager sudo sed -i 's#pm = dynamic#pm = static#' "$POOL_FILE" - #Make copies and create new socket pools - MAX_POOLS=50 + #Create new socket pools for ((c = 1; c <= MAX_POOLS; c++)); do POOL_NAME="socket$c" - NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf" - sudo cp "$POOL_FILE" "$NEW_POOL_FILE" - - sudo sed -i "s#listen =.*#listen = /run/php/php${PHP_VERSION}-fpm-${POOL_NAME}.sock#" "$NEW_POOL_FILE" - sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE" + POOL_SOCKET="/run/php/php${PHP_VERSION}-fpm-${POOL_NAME}.sock" + copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_SOCKET" "static" done - #Make copies and create HTTP pools - MAX_PORTS=50 + #Create TCP port based pools #Division on 1 is required to convert from float to integer - START_PORT=$(echo "(9000 + $PHP_VERSION * 100)/1" | bc) + START_PORT=$(echo "($MIN_PORT + $PHP_VERSION * 100 + 1)/1" | bc) for ((c = 1; c <= MAX_PORTS; c++)); do - POOL_NAME="http$c" + POOL_NAME="port$c" POOL_PORT=$(echo "($START_PORT + $c)/1" | bc) - NEW_POOL_FILE="$POOL_DIR/${POOL_NAME}.conf" - sudo cp "$POOL_FILE" "$NEW_POOL_FILE" - - sudo sed -i "s#listen =.*#listen = 127.0.0.1:$POOL_PORT#" "$NEW_POOL_FILE" - sudo sed -i "s#\[www\]#[$POOL_NAME]#" "$NEW_POOL_FILE" + copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_PORT" "static" done + #Create TCP IPv4 localhost pool + POOL_NAME="localhost" + POOL_PORT=$(echo "($MIN_PORT + $PHP_VERSION * 100)/1" | bc) + POOL_SOCKET="127.0.0.1:$POOL_PORT" + copyPool "$POOL_FILE" "$POOL_NAME" "$POOL_SOCKET" "static" + sudo service "php${PHP_VERSION}-fpm" restart } +setupPools() { + PHP_LIST=$(find /etc/php/ -name 'www.conf' -type f) + while IFS= read -r pool; do + if [[ -n $pool ]]; then + setupPool "$pool" + fi + done <<<"$PHP_LIST" +} + +getNumberOfPHPVersions() { + PHP_COUNT=$(find /etc/php/ -name 'www.conf' -type f | wc -l) + echo "$PHP_COUNT" +} + getAnySocket() { #Get any socket of PHP-FPM: PHP_FIRST=$(find /etc/php/ -name 'www.conf' -type f | head -n1) @@ -60,6 +99,10 @@ getAnyPort() { oneTimeSetUp() { echo "Started job $TRAVIS_JOB_NAME" + echo "Host info:" + nslookup localhost + sudo ifconfig + sudo cat /etc/hosts echo "Copying Zabbix files..." #Install files: sudo cp "$TRAVIS_BUILD_DIR/zabbix/zabbix_php_fpm_discovery.sh" "/etc/zabbix" @@ -75,12 +118,7 @@ oneTimeSetUp() { echo "Setup PHP-FPM..." #Setup PHP-FPM pools: - PHP_LIST=$(find /etc/php/ -name 'www.conf' -type f) - while IFS= read -r pool; do - if [[ -n $pool ]]; then - setupPool "$pool" - fi - done <<<"$PHP_LIST" + setupPools echo "All done, starting tests..." } @@ -116,6 +154,7 @@ testStatusScriptSocket() { DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status") IS_OK=$(echo "$DATA" | grep -F '{"pool":"') assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" + echo "Success test of $PHP_POOL" } testStatusScriptPort() { @@ -126,6 +165,7 @@ testStatusScriptPort() { DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_status.sh" "$PHP_POOL" "/php-fpm-status") IS_OK=$(echo "$DATA" | grep -F '{"pool":"') assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" + echo "Success test of $PHP_POOL" } testZabbixStatusSocket() { @@ -134,6 +174,7 @@ testZabbixStatusSocket() { DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"]) IS_OK=$(echo "$DATA" | grep -F '{"pool":"') assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" + echo "Success test of $PHP_POOL" } testZabbixStatusPort() { @@ -143,6 +184,7 @@ testZabbixStatusPort() { DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.status["$PHP_POOL","/php-fpm-status"]) IS_OK=$(echo "$DATA" | grep -F '{"pool":"') assertNotNull "Failed to get status from pool $PHP_POOL: $DATA" "$IS_OK" + echo "Success test of $PHP_POOL" } testDiscoverScriptReturnsData() { @@ -151,5 +193,38 @@ testDiscoverScriptReturnsData() { assertNotNull "Discover script failed: $DATA" "$IS_OK" } +testDiscoverScriptDebug() { + DATA=$(sudo bash "/etc/zabbix/zabbix_php_fpm_discovery.sh" "debug" "/php-fpm-status") + ERRORS_LIST=$(echo "$DATA" | grep -F 'Error:') + assertNull "Discover script errors: $DATA" "$ERRORS_LIST" +} + +testZabbixDiscoverReturnsData() { + DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"]) + IS_OK=$(echo "$DATA" | grep -F '{"data":[{"{#POOLNAME}"') + assertNotNull "Discover script failed: $DATA" "$IS_OK" +} + +testZabbixDiscoverNumberOfSocketPools() { + DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"]) + NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"socket1",' | wc -l) + PHP_COUNT=$(getNumberOfPHPVersions) + assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS" +} + +testZabbixDiscoverNumberOfIPPools() { + DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"]) + NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"localhost",' | wc -l) + PHP_COUNT=$(getNumberOfPHPVersions) + assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS" +} + +testZabbixDiscoverNumberOfPortPools() { + DATA=$(zabbix_get -s 127.0.0.1 -p 10050 -k php-fpm.discover["/php-fpm-status"]) + NUMBER_OF_POOLS=$(echo "$DATA" | grep -o -F '{"{#POOLNAME}":"port1",' | wc -l) + PHP_COUNT=$(getNumberOfPHPVersions) + assertEquals "Number of pools mismatch" "$PHP_COUNT" "$NUMBER_OF_POOLS" +} + # Load shUnit2. . shunit2 diff --git a/zabbix/zabbix_php_fpm_discovery.sh b/zabbix/zabbix_php_fpm_discovery.sh index 8f18bc8..b322a61 100644 --- a/zabbix/zabbix_php_fpm_discovery.sh +++ b/zabbix/zabbix_php_fpm_discovery.sh @@ -7,6 +7,7 @@ S_PS=$(type -P ps) S_GREP=$(type -P grep) S_AWK=$(type -P awk) S_SORT=$(type -P sort) +S_UNIQ=$(type -P uniq) S_HEAD=$(type -P head) S_LSOF=$(type -P lsof) S_JQ=$(type -P jq) @@ -33,6 +34,10 @@ if [[ ! -f $S_SORT ]]; then ${S_ECHO} "Utility 'sort' not found. Please, install it first." exit 1 fi +if [[ ! -f $S_UNIQ ]]; then + ${S_ECHO} "Utility 'uniq' not found. Please, install it first." + exit 1 +fi if [[ ! -f $S_HEAD ]]; then ${S_ECHO} "Utility 'head' not found. Please, install it first." exit 1 @@ -133,6 +138,7 @@ function ProcessPool() { POOL_NAME=$1 POOL_SOCKET=$2 if [[ -z ${POOL_NAME} ]] || [[ -z ${POOL_SOCKET} ]]; then + PrintDebug "Invalid arguments for ProcessPool" return 0 fi @@ -215,14 +221,21 @@ fi mapfile -t PS_LIST < <($S_PS ax | $S_GREP -F "php-fpm: pool " | $S_GREP -F -v "grep") # shellcheck disable=SC2016 -POOL_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_AWK '{print $NF}' | $S_SORT -u) +POOL_NAMES_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_AWK '{print $NF}' | $S_SORT -u) POOL_FIRST=0 #We store the resulting JSON data for Zabbix in the following var: RESULT_DATA="{\"data\":[" while IFS= read -r line; do # shellcheck disable=SC2016 - POOL_PID=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_GREP -F -w "php-fpm: pool $line" | $S_HEAD -1 | $S_AWK '{print $1}') - if [[ -n $POOL_PID ]]; then + POOL_PID_LIST=$(${S_PRINTF} '%s\n' "${PS_LIST[@]}" | $S_GREP -F -w "php-fpm: pool $line" | $S_AWK '{print $1}') + POOL_PID_ARGS="" + while IFS= read -r POOL_PID; do + if [[ -n $POOL_PID ]]; then + POOL_PID_ARGS="$POOL_PID_ARGS -p $POOL_PID" + fi + done <<<"$POOL_PID_LIST" + + if [[ -n $POOL_PID_ARGS ]]; then #We search for socket or IP address and port #Socket example: #php-fpm7. 25897 root 9u unix 0x000000006509e31f 0t0 58381847 /run/php/php7.3-fpm.sock type=STREAM @@ -235,84 +248,85 @@ while IFS= read -r line; do #php-fpm7. 1203 www-data 8u IPv4 15070917 0t0 TCP localhost.localdomain:23054->localhost.localdomain:postgresql (ESTABLISHED) #More info at https://github.com/rvalitov/zabbix-php-fpm/issues/12 - PrintDebug "Started analysis of pool $line, PID $POOL_PID" + PrintDebug "Started analysis of pool $line, PID(s): $POOL_PID_ARGS" #Extract only important information: #Use -P to show port number instead of port name, see https://github.com/rvalitov/zabbix-php-fpm/issues/24 - POOL_PARAMS_LIST=$($S_LSOF -P -p "$POOL_PID" 2>/dev/null | $S_GREP -w -e "unix" -e "TCP") + #Use -n flag to show IP address and not convert it to domain name (like localhost) + #Sometimes different PHP-FPM versions may have the same names of pools, so we need to consider that. + # It's considered that a pair of pool name and socket must be unique. + #Sorting is required, because uniq needs it + POOL_PARAMS_LIST=$($S_LSOF -n -P $POOL_PID_ARGS 2>/dev/null | $S_GREP -w -e "unix" -e "TCP" | $S_SORT -u | $S_UNIQ -f8) FOUND_POOL="" while IFS= read -r pool; do if [[ -n $pool ]]; then - if [[ -z $FOUND_POOL ]]; then - PrintDebug "Checking process: $pool" - # shellcheck disable=SC2016 - POOL_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $5}') - # shellcheck disable=SC2016 - POOL_SOCKET=$(${S_ECHO} "${pool}" | $S_AWK '{print $9}') - if [[ -n $POOL_TYPE ]] && [[ -n $POOL_SOCKET ]]; then - if [[ $POOL_TYPE == "unix" ]]; then - #We have a socket here, test if it's actually a socket: - if [[ -S $POOL_SOCKET ]]; then - PrintDebug "Found socket $POOL_SOCKET" + PrintDebug "Checking process: $pool" + # shellcheck disable=SC2016 + POOL_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $5}') + # shellcheck disable=SC2016 + POOL_SOCKET=$(${S_ECHO} "${pool}" | $S_AWK '{print $9}') + if [[ -n $POOL_TYPE ]] && [[ -n $POOL_SOCKET ]]; then + if [[ $POOL_TYPE == "unix" ]]; then + #We have a socket here, test if it's actually a socket: + if [[ -S $POOL_SOCKET ]]; then + PrintDebug "Found socket $POOL_SOCKET" + ProcessPool "${line}" "${POOL_SOCKET}" + POOL_STATUS=$? + if [[ ${POOL_STATUS} -gt 0 ]]; then + FOUND_POOL="1" + PrintDebug "Success: socket $POOL_SOCKET returned valid status data" + EncodeToJson "${line}" "${POOL_SOCKET}" + else + PrintDebug "Error: socket $POOL_SOCKET didn't return valid data" + fi + else + PrintDebug "Error: specified socket $POOL_SOCKET is not valid" + fi + elif [[ $POOL_TYPE == "IPv4" ]] || [[ $POOL_TYPE == "IPv6" ]]; then + #We have a TCP connection here, check it: + # shellcheck disable=SC2016 + CONNECTION_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $8}') + if [[ $CONNECTION_TYPE == "TCP" ]]; then + #The connection must have state LISTEN: + LISTEN=$(${S_ECHO} "${pool}" | $S_GREP -F -w "(LISTEN)") + if [[ -n $LISTEN ]]; then + #Check and replace * to localhost if it's found. Asterisk means that the PHP listens on + #all interfaces. + PrintDebug "Found TCP connection $POOL_SOCKET" + POOL_SOCKET=${POOL_SOCKET/\*:/localhost:} + PrintDebug "Processed TCP connection $POOL_SOCKET" ProcessPool "${line}" "${POOL_SOCKET}" POOL_STATUS=$? if [[ ${POOL_STATUS} -gt 0 ]]; then FOUND_POOL="1" - PrintDebug "Success: socket $POOL_SOCKET returned valid status data" + PrintDebug "Success: TCP connection $POOL_SOCKET returned valid status data" + EncodeToJson "${line}" "${POOL_SOCKET}" else - PrintDebug "Error: socket $POOL_SOCKET didn't return valid data" + PrintDebug "Error: TCP connection $POOL_SOCKET didn't return valid data" fi else - PrintDebug "Error: specified socket $POOL_SOCKET is not valid" - fi - elif [[ $POOL_TYPE == "IPv4" ]] || [[ $POOL_TYPE == "IPv6" ]]; then - #We have a TCP connection here, check it: - # shellcheck disable=SC2016 - CONNECTION_TYPE=$(${S_ECHO} "${pool}" | $S_AWK '{print $8}') - if [[ $CONNECTION_TYPE == "TCP" ]]; then - #The connection must have state LISTEN: - LISTEN=$(${S_ECHO} "${pool}" | $S_GREP -F -w "(LISTEN)") - if [[ -n $LISTEN ]]; then - #Check and replace * to localhost if it's found. Asterisk means that the PHP listens on - #all interfaces. - POOL_SOCKET=$(${S_ECHO} -n "${POOL_SOCKET/*:/localhost:}") - PrintDebug "Found TCP connection $POOL_SOCKET" - ProcessPool "${line}" "${POOL_SOCKET}" - POOL_STATUS=$? - if [[ ${POOL_STATUS} -gt 0 ]]; then - FOUND_POOL="1" - PrintDebug "Success: TCP connection $POOL_SOCKET returned valid status data" - else - PrintDebug "Error: TCP connection $POOL_SOCKET didn't return valid data" - fi - else - PrintDebug "Warning: expected connection state must be LISTEN, but it was not detected" - fi - else - PrintDebug "Warning: expected connection type is TCP, but found $CONNECTION_TYPE" + PrintDebug "Warning: expected connection state must be LISTEN, but it was not detected" fi else - PrintDebug "Unsupported type $POOL_TYPE, skipping" + PrintDebug "Warning: expected connection type is TCP, but found $CONNECTION_TYPE" fi else - PrintDebug "Warning: pool type or socket is empty" + PrintDebug "Unsupported type $POOL_TYPE, skipping" fi else - PrintDebug "Pool already found, skipping process: $pool" + PrintDebug "Warning: pool type or socket is empty" fi else PrintDebug "Error: failed to get process information. Probably insufficient privileges. Use sudo or run this script under root." fi done <<<"$POOL_PARAMS_LIST" - if [[ -n ${FOUND_POOL} ]]; then - EncodeToJson "${line}" "${POOL_SOCKET}" - else + if [[ -z ${FOUND_POOL} ]]; then PrintDebug "Error: failed to discover information for pool $line" fi else PrintDebug "Error: failed to find PID for pool $line" fi -done <<<"$POOL_LIST" +done <<<"$POOL_NAMES_LIST" PrintDebug "Processing pools from old cache..." for CACHE_ITEM in "${CACHE[@]}"; do