Merge pull request #32 from ivanilves/testing/advanced-cases

Integration testing: Cases (advanced)

.travis.yml

@@ -15,10 +15,18 @@ install:
   - docker pull ${DOCKER_REPO}/xiri-centos
 
 script:
-  - testing/integration.sh zero
-  - testing/integration.sh basic_connectivity_with_ssh
-  - testing/integration.sh connect_with_ssh_agent
-  - testing/integration.sh connect_with_ssh_key
+  - testing/run_integration_case.sh zero
+  - testing/run_integration_case.sh basic_connectivity_with_ssh
+  - testing/run_integration_case.sh connect_with_ssh_agent
+  - testing/run_integration_case.sh connect_with_ssh_key
+  - testing/run_integration_case.sh do_client_teardown
+  - testing/run_integration_case.sh do_server_teardown
+  - testing/run_integration_case.sh run_with_dns_propagation
+  - testing/run_integration_case.sh run_without_dns_propagation
+  - testing/run_integration_case.sh run_with_reconnection
+  - testing/run_integration_case.sh run_without_reconnection
+  - testing/run_integration_case.sh run_with_route_discovery
+  - testing/run_integration_case.sh run_without_route_discovery
 
 branches:
   only:

scripts/client-ping-server.sh

@@ -15,4 +15,4 @@ declare -r IP_BASE=${2}
 let SERVER_LAST_IP_ADDR_OCTET="4*(${TUNNEL_ID}-1)+2"
 declare -r SERVER_IP_ADDR=${IP_BASE}.${SERVER_LAST_IP_ADDR_OCTET}
 
-ping -c1 -nq ${SERVER_IP_ADDR} >/dev/null
+ping -W3 -c1 -nq ${SERVER_IP_ADDR} >/dev/null

scripts/server-execute.sh

@@ -32,7 +32,7 @@ echo "CONNECTED"
 
 FAILED_PINGS=0
 while [[ ${FAILED_PINGS} -lt ${MAX_FAILED_PINGS} ]]; do
-  ping -c3 -nq ${CLIENT_IP_ADDR} >/dev/null
+  ping -W3 -c1 -nq ${CLIENT_IP_ADDR} >/dev/null
   if [[ ${?} -ne 0 ]]; then
     let FAILED_PINGS+=1
     logger -i -p warn "xiringuito[${TUNNEL_ID}]: Failed to ping ${CLIENT_IP_ADDR} (${FAILED_PINGS}/${MAX_FAILED_PINGS})"

testing/integration/cases/connect_with_ssh_agent.sh

@@ -1,9 +1,3 @@
 export EXIT_AFTER_CONNECT=1
 
-eval `ssh-agent -s`
-
-ssh-add ssh-keys/id_rsa
-
 ${XIRI_EXE} -X ${SSH_USER}@${REMOTE_IP}
-
-kill ${SSH_AGENT_PID}

testing/integration/cases/connect_with_ssh_key.sh

@@ -1,3 +1,5 @@
+unset SSH_AUTH_SOCK
+
 export SSH_EXTRA_OPTS="${SSH_EXTRA_OPTS} -i ${PWD}/ssh-keys/id_rsa"
 export EXIT_AFTER_CONNECT=1
 

testing/integration/cases/do_client_teardown.sh

@@ -0,0 +1,29 @@
+function get_client_links(){
+  ip link | egrep "tun[0-9]{1,2}" | cut -f2 -d' ' | tr '\n' ' '
+}
+
+INIT_DELAY=2
+DOWN_DELAY=2
+
+ORIG_LINKS=$(get_client_links)
+
+${XIRI_EXE} -X -R ${SSH_USER}@${REMOTE_IP} &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+NEW_LINKS=$(get_client_links)
+
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}
+FINAL_LINKS=$(get_client_links)
+
+if [[ "${ORIG_LINKS}" == "${NEW_LINKS}" ]]; then
+  complain "TUNx link state should change after xiringuito started and connected!"
+  complain "* Original : ${ORIG_LINKS}"
+  complain "* New      : ${NEW_LINKS}"
+  exit 1
+fi
+
+if [[ "${ORIG_LINKS}" != "${FINAL_LINKS}" ]]; then
+  complain "TUNx link state should be restored after xiringuito stop!"
+  complain "* Original : ${ORIG_LINKS}"
+  complain "* Final    : ${FINAL_LINKS}"
+  exit 1
+fi

testing/integration/cases/do_server_teardown.sh

@@ -0,0 +1,31 @@
+function get_server_links(){
+  ssh -i ssh-keys/id_rsa ${SSH_EXTRA_OPTS} ${SSH_USER}@${REMOTE_IP} ip link \
+    | egrep "tun[0-9]{1,2}" | cut -f2 -d' ' | tr '\n' ' '
+}
+
+INIT_DELAY=10
+DOWN_DELAY=10
+
+ORIG_LINKS=$(get_server_links)
+
+${XIRI_EXE} -f 1 -X -R ${SSH_USER}@${REMOTE_IP} &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+NEW_LINKS=$(get_server_links)
+
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}
+
+FINAL_LINKS=$(get_server_links)
+
+if [[ "${ORIG_LINKS}" == "${NEW_LINKS}" ]]; then
+  complain "TUNx link state should change after xiringuito started and connected!"
+  complain "* Original : ${ORIG_LINKS}"
+  complain "* New      : ${NEW_LINKS}"
+  exit 1
+fi
+
+if [[ "${ORIG_LINKS}" != "${FINAL_LINKS}" ]]; then
+  complain "TUNx link state should be restored after xiringuito stop!"
+  complain "* Original : ${ORIG_LINKS}"
+  complain "* Final    : ${FINAL_LINKS}"
+  exit 1
+fi

testing/integration/cases/run_with_dns_propagation.sh

@@ -0,0 +1,18 @@
+INIT_DELAY=20
+DOWN_DELAY=20
+
+ORIG_RESOLV_CONF=$(cat /etc/resolv.conf)
+warn "${ORIG_RESOLV_CONF}"
+
+${XIRI_EXE} -f 1 -X -R ${SSH_USER}@${REMOTE_IP} 10.245.245.245/32 &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+NEW_RESOLV_CONF=$(cat /etc/resolv.conf)
+warn "${NEW_RESOLV_CONF}"
+
+if [[ "${ORIG_RESOLV_CONF}" == "${NEW_RESOLV_CONF}" ]]; then
+  complain "Should update /etc/resolv.conf"
+  exit 1
+fi
+
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}

testing/integration/cases/run_with_reconnection.sh

@@ -0,0 +1,17 @@
+INIT_DELAY=5
+DOWN_DELAY=5
+
+${XIRI_EXE} -f 1 -X ${SSH_USER}@${REMOTE_IP} 10.245.245.245/32 &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+docker restart xiri-${DIST}
+
+sleep ${INIT_DELAY}
+
+warn "$(ip route | grep 10.245.245.245)"
+if [[ -z "$(ip route | grep 10.245.245.245)" ]]; then
+  complain "Routing not restored after reconnection!"
+  exit 1
+fi
+
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}

testing/integration/cases/run_with_route_discovery.sh

@@ -0,0 +1,30 @@
+INIT_DELAY=15
+DOWN_DELAY=5
+POST_DELAY=5
+
+if [[ -f ${WD}/discover-routes ]]; then
+  mv ${WD}/discover-routes /tmp/discover-routes.orig
+fi
+echo '#!/bin/sh' >${WD}/discover-routes
+echo 'echo ROUTE:10.42.42.42/32' >>${WD}/discover-routes
+chmod +x ${WD}/discover-routes
+
+${XIRI_EXE} -f 1 ${SSH_USER}@${REMOTE_IP} &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+warn "$(ip route | grep 10.42.42.42)"
+if [[ -z "$(ip route | grep 10.42.42.42)" ]]; then
+  complain "Route not added by route discovery: 10.42.42.42/32"
+  exit 1
+fi
+
+rm ${WD}/discover-routes
+if [[ -f /tmp/discover-routes.orig ]]; then
+  mv /tmp/discover-routes.orig ${WD}/discover-routes
+fi
+
+set +e
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}
+set -e
+
+sleep ${POST_DELAY}

testing/integration/cases/run_without_dns_propagation.sh

@@ -0,0 +1,18 @@
+INIT_DELAY=20
+DOWN_DELAY=20
+
+ORIG_RESOLV_CONF=$(cat /etc/resolv.conf)
+warn "${ORIG_RESOLV_CONF}"
+
+${XIRI_EXE} -f 1 -X -R -D ${SSH_USER}@${REMOTE_IP} 10.245.245.245/32 &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+NEW_RESOLV_CONF=$(cat /etc/resolv.conf)
+warn "${NEW_RESOLV_CONF}"
+
+if [[ "${ORIG_RESOLV_CONF}" != "${NEW_RESOLV_CONF}" ]]; then
+  complain "Should NOT update /etc/resolv.conf"
+  exit 1
+fi
+
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}

testing/integration/cases/run_without_reconnection.sh

@@ -0,0 +1,14 @@
+INIT_DELAY=5
+
+${XIRI_EXE} -f 1 -X -R ${SSH_USER}@${REMOTE_IP} 10.245.245.245/32 &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+docker restart xiri-${DIST}
+
+sleep ${INIT_DELAY}
+
+warn "$(ip route | grep 10.245.245.245)"
+if [[ -n "$(ip route | grep 10.245.245.245)" ]]; then
+  complain "Routing not down, but it should be!"
+  exit 1
+fi

testing/integration/cases/run_without_route_discovery.sh

@@ -0,0 +1,26 @@
+INIT_DELAY=15
+DOWN_DELAY=5
+POST_DELAY=5
+
+if [[ -f ${WD}/discover-routes ]]; then
+  mv ${WD}/discover-routes /tmp/discover-routes.orig
+fi
+
+${XIRI_EXE} -f 1 -X ${SSH_USER}@${REMOTE_IP} &
+XIRI_PID=${!}; sleep ${INIT_DELAY}
+
+warn "$(ip route | grep 10.42.42.42)"
+if [[ -n "$(ip route | grep 10.42.42.42)" ]]; then
+  complain "Route added by (non-existing) route discovery: 10.42.42.42/32"
+  exit 1
+fi
+
+if [[ -f /tmp/discover-routes.orig ]]; then
+  mv /tmp/discover-routes.orig ${WD}/discover-routes
+fi
+
+set +e
+kill_reliably ${XIRI_PID} ${DOWN_DELAY}
+set -e
+
+sleep ${POST_DELAY}

testing/run_integration_case.sh

@@ -17,12 +17,42 @@ function setup(){
   done
 }
 
+function kill_reliably(){
+  local TARGET_PID=${1}
+  local CHECK_DELAY=${2}
+
+  kill ${TARGET_PID}
+  sleep ${CHECK_DELAY}
+  if [[ $(ps -p ${TARGET_PID} | wc -l) -eq 2 ]]; then
+    kill -9 ${TARGET_PID} &>/dev/null
+    sleep ${CHECK_DELAY}
+  fi
+}
+
+function warn(){
+  local LC=$(echo "${@}" | wc -l)
+  local CL=1
+
+  while [[ ${CL} -le ${LC} ]]; do
+    echo -e "\033[1;33m$(echo "${@}" | head -n${CL} | tail -n1)\033[0m"
+    let CL+=1
+  done
+}
+
+function complain(){
+  echo -e "\033[1;31m>>> ${@}\033[0m"
+}
+
 function run_case(){
   declare -r CASE=${1}
 
-  declare -r XIRI_EXE=../../xiringuito
+  pushd ../.. >/dev/null; WD=${PWD}; popd >/dev/null
+
+  declare -r XIRI_EXE=${WD}/xiringuito
   declare -r SSH_USER=root
 
+  eval `ssh-agent -s`; ssh-add ssh-keys/id_rsa
+
   for DIST in ${DISTS}; do
     echo
     echo "[ RUN: ${1} / ${DIST} ]"
@@ -34,9 +64,11 @@ function run_case(){
     export LANG=C
     export LC_ALL=C
 
+    [[ ${DEBUG} ]] && set -x
     set -e
     source cases/${1}.sh
     set +e
+    [[ ${DEBUG} ]] && set +x
   done
 }
 
@@ -49,10 +81,13 @@ function teardown(){
     make docker-rm DIST=${DIST}
   done
 
+  kill ${SSH_AGENT_PID}
+
   if [[ "${SUCCESS}" == "true" ]]; then
     echo
     echo -e "\033[0;32m[ OK ]\033[0m"
     echo
+    sleep 1
     exit 0
   fi
 
@@ -65,6 +100,8 @@ function teardown(){
 if [[ ${#} != 1 ]]; then
   echo "Usage: $(basename ${0}) CASE"
   echo
+  echo "HINT: Set 'DEBUG' environment variable to see case execution trace."
+  echo
   echo "Available integration testing cases:"
   echo "${CASES}"
   exit 1