run tests with run_test.sh; automatically generates tests per each running container, if needed

bats.Dockerfile

@@ -7,11 +7,13 @@ MAINTAINER Alexei Ledenev <alexei.led@gmail.com>
 ENV VERSION 1.10.0
 ENV BATS_VERSION 0.4.0
 
+LABEL docker_bench_security=true
+
 WORKDIR /usr/bin
 
 RUN apk update && \
     apk upgrade && \
-    apk --update add curl bash && \
+    apk --update add curl bash ncurses ncurses-terminfo && \
     curl -sS https://get.docker.com/builds/Linux/x86_64/docker-$VERSION > docker-$VERSION && \
     curl -sS https://get.docker.com/builds/Linux/x86_64/docker-$VERSION.sha256 > docker-$VERSION.sha256 && \
     sha256sum -c docker-$VERSION.sha256 && \
@@ -28,7 +30,11 @@ RUN curl -o "/tmp/v${BATS_VERSION}.tar.gz" -L \
 RUN mkdir /docker-bench-security
 
 COPY . /docker-bench-security
+RUN chmod +x /docker-bench-security/run_tests.sh
 
 WORKDIR /docker-bench-security
 
-ENTRYPOINT ["/usr/local/bin/bats", "/docker-bench-security/test"]
+VOLUME /var/docker-bench
+
+CMD ["-r"]
+ENTRYPOINT ["./run_tests.sh"]

generate_tests.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+. ./helper_lib.sh
+
+TEST_SRC=./test
+BENCH_ROOT=/var/docker-bench
+TEST_ROOT=$BENCH_ROOT/test
+
+prepare_tests_directory()
+{
+  mkdir -p $BENCH_ROOT
+  if [ -d "$TEST_ROOT" ]; then
+    rm -rf $TEST_ROOT
+  fi
+  if [ ! -f "$BENCH_ROOT/helper_lib.sh" ]; then
+    cp helper_lib.sh $BENCH_ROOT
+  fi
+  cp -r $TEST_SRC $TEST_ROOT
+}
+
+list_running_containers() {
+    # List all running containers
+  local containers=$(docker ps | sed '1d' | awk '{print $NF}')
+  # If there is a container with label docker_bench_security, memorize it:
+  local benchcont="nil"
+  for c in $containers; do
+    labels=$(docker inspect --format '{{ .Config.Labels }}' "$c")
+    contains "$labels" "docker_bench_security" && benchcont="$c"
+  done
+  # List all running containers except docker-bench (use names to improve readability in logs)
+  docker ps -aq --format="{{.Names}}" | grep -v "$benchcont" | tr "\n" " "
+}
+
+generate_all_tests() {
+  # prepare test direcory: copy tests and templates
+  prepare_tests_directory
+  # generate tests from templates for running containers
+  local containers=$(list_running_containers)
+  ( cd $TEST_ROOT
+  for c in ${containers[@]}; do
+    for t in *.bats.template; do
+      sed -e "s/{{c}}/$c/g" "${t}" > "${t%.*.*}_${c}.bats"
+    done
+  done
+  )
+}

run_tests.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+. ./generate_tests.sh
+
+TERMINFO=/usr/share/terminfo
+
+TEST_RESULTS=$BENCH_ROOT/results
+
+# make result folder (inside VOLUME)
+# mkdir -p $TEST_RESULTS
+# generate all tests: copy host and daemon level tests and generate container level tests for running containers
+# generate_all_tests
+# run bats with all tests or passed tests
+# bats $TEST_ROOT >
+
+#Set Script Name variable
+SCRIPT="run_tests.sh"
+
+#Initialize variables to default values.
+OPT_FORMAT="t"
+OPT_OUTPUT=$TEST_RESULTS
+OPT_RESULTS=1
+
+#Set fonts for Help.
+BOLD=`tput bold`
+REV=`tput smso`
+NORM=`tput sgr0`
+
+#Help function
+HELP() {
+  echo -e \\n"Help documentation for ${BOLD}${SCRIPT}${NORM}"\\n
+  echo -e "Basic usage: ${BOLD}$SCRIPT [-c] [-p|-t] [-o path] <test> [<test> ...]${NORM}"\\n
+  echo -e "Command line switches are optional. The following switches are recognized."
+  echo -e "${REV}-c${NORM}  --Displays number of tests. No further functions are performed."
+  echo -e "${REV}-g${NORM}  --Generates all CIS Bats tests without execution. No further functions are performed."
+  echo -e "${REV}-p${NORM}  --Show results in pretty format."
+  echo -e "${REV}-t${NORM}  --Show results in TAP format. This is the default format."
+  echo -e "${REV}-t${NORM}  --Create test results files: ${BOLD}tests_<timestamp>.tap${NORM} in test result folder."
+  echo -e "${REV}-o${NORM}  --Specify test result folder. Default to ${BOLD}$TEST_RESULTS${NORM}."
+  echo -e "${REV}-h${NORM}  --Displays this help message. No further functions are performed."\\n
+  echo -e "Example: ${BOLD}$SCRIPT -t -o $TEST_RESULTS${NORM}"\\n
+  exit 1
+}
+
+#Check the number of arguments. If none are passed, print help and exit.
+NUMARGS=$#
+if [ $NUMARGS -eq 0 ]; then
+  HELP
+fi
+
+### Start getopts code ###
+
+#Parse command line flags
+while getopts o:rptcgh FLAG; do
+  case $FLAG in
+    o)  # output test results into specified folder
+      OPT_OUTPUT=$OPTARG
+      ;;
+    p)  # output test results in TAP format
+      OPT_FORMAT="p"
+      ;;
+    t)  # output test results in pretty format
+      OPT_FORMAT="t"
+      ;;
+    r)  # save test results into file
+      OPT_RESULTS=0
+      ;;
+    c)  # count tests
+      if [ -d "$TEST_ROOT" ]; then
+        echo -e "There are ${BOLD}$(bats $TEST_ROOT -c)${NORM} tests in ${BOLD}$TEST_ROOT${NORM}"
+      else
+        echo -e "No tests found, run ${BOLD}${SCRIPT}${NORM} with ${REV}-g${NORM} option first."
+      fi
+      exit 1
+      ;;
+    g)  # genetate all Bats tests: copy tests and generate tests (per container) from templates
+      generate_all_tests
+      exit 1
+      ;;
+    h)  #show help
+      HELP
+      ;;
+    \?) #unrecognized option - show help
+      echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed."
+      HELP
+      ;;
+  esac
+done
+
+shift $((OPTIND-1))  #This tells getopts to move on to the next argument.
+
+### End getopts code ###
+
+### Run Bats tests ###
+
+TESTS=$TEST_ROOT
+if [ ! -d $TEST_ROOT ]; then # generate tests if needed
+  generate_all_tests
+fi
+
+if [ $# -ne 0 ]; then # get tests from command line
+  TESTS=$*
+fi
+
+if [ $OPT_RESULTS -eq 0 ]; then # run tests and [create test result file]
+  if [ ! -d "$OPT_OUTPUT" ]; then
+    mkdir -p "$OPT_OUTPUT"
+  fi
+  bats $TESTS -${OPT_FORMAT} > "${OPT_OUTPUT}/tests_$(date +%s).tap"
+else
+  bats $TESTS -${OPT_FORMAT}
+fi
+
+exit 0

test/1_host_configuration.bats

@@ -70,9 +70,8 @@ load "$BATS_TEST_DIRNAME/../helper_lib.sh"
   assert_success
 }
 
-# 1.8
+test_audit_directory() {
-@test "1.8  - Audit Docker files and directories - /var/lib/docker" {
+  local directory="$1"
-  directory="/var/lib/docker"
   assert [ -d "$directory" ]
   run command -v auditctl >/dev/null
   assert_success
@@ -80,72 +79,51 @@ load "$BATS_TEST_DIRNAME/../helper_lib.sh"
   assert_success
 }
 
-# 1.9
+test_audit_file() {
-@test "1.9  - Audit Docker files and directories - /etc/docker" {
+  file="$1"
-  directory="/etc/docker"
+  assert [ -f "$file" ]
-  assert [ -d "$directory" ]
   run command -v auditctl
   assert_success
-  run auditctl -l | grep $directory
+  run auditctl -l | grep "$file"
   assert_success
 }
 
+# 1.8
+@test "1.8  - Audit Docker files and directories - /var/lib/docker" {
+  test_audit_directory "/var/lib/docker"
+}
+
+# 1.9
+@test "1.9  - Audit Docker files and directories - /etc/docker" {
+  test_audit_directory "/etc/docker"
+}
+
 # 1.10
 @test "1.10 - Audit Docker files and directories - docker.service" {
-  file="$(get_systemd_service_file docker.service)"
+  test_audit_file "$(get_systemd_service_file docker.service)"
-  assert [ -f "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep "$file"
-  assert_success
 }
 
 # 1.11
 @test "1.11 - Audit Docker files and directories - docker.socket" {
-  file="$(get_systemd_service_file docker.socket)"
+  test_audit_file "$(get_systemd_service_file docker.socket)"
-  assert [ -e "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep "$file"
-  assert_success
 }
 
 # 1.12
 @test "1.12 - Audit Docker files and directories - /etc/default/docker" {
-  file="/etc/default/docker"
+  test_audit_file "/etc/default/docker"
-  assert [ -f "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep $file
-  assert_success
 }
 
 # 1.13
 @test "1.13 - Audit Docker files and directories - /etc/docker/daemon.json" {
-  file="/etc/docker/daemon.json"
+  test_audit_file "/etc/docker/daemon.json"
-  assert [ -f "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep $file
-  assert_success
 }
 
 # 1.14
 @test "1.14 - Audit Docker files and directories - /usr/bin/docker-containerd" {
-  file="/usr/bin/docker-containerd"
+  test_audit_file "/usr/bin/docker-containerd"
-  assert [ -f "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep $file
-  assert_success
 }
 
 # 1.15
 @test "1.15 - Audit Docker files and directories - /usr/bin/docker-runc" {
-  file="/usr/bin/docker-runc"
+  test_audit_file "/usr/bin/docker-runc"
-  assert [ -f "$file" ]
-  run command -v auditctl
-  assert_success
-  run auditctl -l | grep $file
-  assert_success
 }

test/4_1_create_user_in_container.bats.template

@@ -0,0 +1,18 @@
+#!/usr/bin/env bats
+
+load "test_helper/bats-support/load"
+load "test_helper/bats-assert/load"
+load "$BATS_TEST_DIRNAME/../helper_lib.sh"
+
+
+# 4.1
+@test "4.1  - Create a user for the container {{c}}" {
+  local user=$(docker inspect --format 'User={{.Config.User}}' "{{c}}")
+  if [ "$user" = "User=" -o "$user" = "User=[]" -o "$user" = "User=<no value>" ]; then
+    # get PID 1 and check if it's running as root (uid=0)
+    local uid=$(docker exec {{c}} awk '/^Uid:/{print $2}' /proc/1/status)
+    if [ $uid -eq 0 ]; then
+      fail "Running as root: {{c}}"
+    fi
+  fi
+}

test/4_container_images.bats

@@ -0,0 +1,9 @@
+#!/usr/bin/env bats
+
+load "test_helper/bats-support/load"
+load "test_helper/bats-assert/load"
+
+# 4.5
+@test "4.5  - Enable Content trust for Docker" {
+  assert [ "x$DOCKER_CONTENT_TRUST" = "x1" ]
+}