Refactored auto-recording function

fix file error when start/stop/start container

.gitignore

@@ -7,3 +7,6 @@
 coverage.xml
 xunit.xml
 coverage/*
+
+# Video
+video*

.travis.yml

@@ -11,12 +11,12 @@ services:
 install: "pip install -r requirements.txt"
 
 env:
-  matrix:
+  - ANDROID_VERSION=5.0.1
-    - ANDROID_VERSION=5.0.1
+  - ANDROID_VERSION=5.1.1
-    - ANDROID_VERSION=5.1.1
+  - ANDROID_VERSION=6.0
-    - ANDROID_VERSION=6.0
+  - ANDROID_VERSION=7.0
-    - ANDROID_VERSION=7.0
+  - ANDROID_VERSION=7.1.1
-    - ANDROID_VERSION=7.1.1
+  - REAL_DEVICE=True
 
 script: bash travis.sh
 

README.md

@@ -1,6 +1,7 @@
 Docker-Android
 ==============
 
+[![Join the chat at https://gitter.im/butomo1989/docker-android](https://badges.gitter.im/butomo1989/docker-android.svg)](https://gitter.im/butomo1989/docker-android?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Build Status](https://travis-ci.org/butomo1989/docker-android.svg?branch=master)](https://travis-ci.org/butomo1989/docker-android)
 [![codecov](https://codecov.io/gh/butomo1989/docker-android/branch/master/graph/badge.svg)](https://codecov.io/gh/butomo1989/docker-android)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/3f000ffb97db45a59161814e1434c429)](https://www.codacy.com/app/butomo1989/docker-appium?utm_source=github.com&utm_medium=referral&utm_content=butomo1989/docker-appium&utm_campaign=Badge_Grade)

docker-compose.yml

@@ -14,18 +14,20 @@ services:
   nexus_7.1.1:
     image: butomo1989/docker-android-x86-7.1.1
     privileged: true
-    # Change path of apk that you want to test. I use sample_apk that I provide in folder "example"
-    volumes:
-      - $PWD/example/sample_apk:/root/tmp
     # Increase scale number if needed
     scale: 1
     ports:
       - 6080
+    # Change path of apk that you want to test. I use sample_apk that I provide in folder "example"
+    volumes:
+      - $PWD/example/sample_apk:/root/tmp
+      - ./video-nexus_7.1.1:/tmp/video
     environment:
       - DEVICE=Nexus 5
       - CONNECT_TO_GRID=True
       - APPIUM=true
       - SELENIUM_HOST=selenium_hub
+      - AUTO_RECORD=True
 
   # Docker-Android for mobile website testing with chrome browser
   # Chrome browser exists only for version 7.0 and 7.1.1
@@ -36,12 +38,15 @@ services:
     scale: 1
     ports:
       - 6080
+    volumes:
+      - ./video-samsung_7.1.1:/tmp/video
     environment:
       - DEVICE=Samsung Galaxy S6
       - CONNECT_TO_GRID=True
       - APPIUM=true
       - SELENIUM_HOST=selenium_hub
       - MOBILE_WEB_TEST=True
+      - AUTO_RECORD=True
 
   # Docker-Android for mobile website testing with default browser
   # Default browser exists only for version 5.0.1, 5.1.1 and 6.0
@@ -52,9 +57,12 @@ services:
     scale: 1
     ports:
       - 6080
+    volumes:
+      - ./video-samsung_5.1.1:/tmp/video
     environment:
       - DEVICE=Samsung Galaxy S6
       - CONNECT_TO_GRID=True
       - APPIUM=true
       - SELENIUM_HOST=selenium_hub
       - MOBILE_WEB_TEST=True
+      - AUTO_RECORD=True

docker/Emulator_arm

@@ -29,6 +29,13 @@ WORKDIR /root
 #   Numpy, For faster performance: https://github.com/novnc/websockify/issues/77
 # net-tools
 #   Netstat
+#------------------
+#  Video Recording
+#------------------
+# ffmpeg
+#   Video recorder
+# jq
+#   Sed for JSON data
 #==================
 RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \
     xterm \
@@ -39,6 +46,8 @@ RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \
     menu \
     python-numpy \
     net-tools \
+    ffmpeg \
+    jq \
  && rm -rf /var/lib/apt/lists/*
 
 #=======
@@ -100,6 +109,7 @@ ENV DISPLAY=:0 \
     LOCAL_PORT=5900 \
     TARGET_PORT=6080 \
     TIMEOUT=1 \
+    VIDEO_PATH=/tmp/video \
     LOG_PATH=/var/log/supervisor
 
 #===============

docker/Emulator_x86

@@ -30,6 +30,13 @@ WORKDIR /root
 # net-tools
 #   Netstat
 #------------------
+#  Video Recording
+#------------------
+# ffmpeg
+#   Video recorder
+# jq
+#   Sed for JSON data
+#------------------
 #    KVM Package
 # for emulator x86
 # https://help.ubuntu.com/community/KVM/Installation
@@ -48,6 +55,8 @@ RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \
     menu \
     python-numpy \
     net-tools \
+    ffmpeg \
+    jq \
     qemu-kvm \
     libvirt-bin \
     ubuntu-vm-builder \
@@ -113,6 +122,7 @@ ENV DISPLAY=:0 \
     LOCAL_PORT=5900 \
     TARGET_PORT=6080 \
     TIMEOUT=1 \
+    VIDEO_PATH=/tmp/video \
     LOG_PATH=/var/log/supervisor
 
 #===============

docker/Real_device

@@ -1 +1,117 @@
-# TODO
+FROM appium/appium:1.6.6-beta-p0
+
+LABEL maintainer "Budi Utomo <budi.ut.1989@gmail.com>"
+
+#=============
+# Set WORKDIR
+#=============
+WORKDIR /root
+
+#==================
+# General Packages
+#------------------
+# xterm
+#   Terminal emulator
+# supervisor
+#   Process manager
+# socat
+#   Port forwarder
+#------------------
+#  NoVNC Packages
+#------------------
+# x11vnc
+#   VNC server for X display
+# openbox
+#   Windows manager
+# menu
+#   Debian menu
+# python-numpy
+#   Numpy, For faster performance: https://github.com/novnc/websockify/issues/77
+# net-tools
+#   Netstat
+#------------------
+#  Video Recording
+#------------------
+# ffmpeg
+#   Video recorder
+# jq
+#   Sed for JSON data
+#==================
+RUN apt-get -qqy update && apt-get -qqy install --no-install-recommends \
+    xterm \
+    supervisor \
+    socat \
+    x11vnc \
+    openbox \
+    menu \
+    python-numpy \
+    net-tools \
+    ffmpeg \
+    jq \
+ && rm -rf /var/lib/apt/lists/*
+
+#=======
+# noVNC
+# Use same commit id that docker-selenium uses
+# https://github.com/elgalu/docker-selenium/blob/236b861177bd2917d864e52291114b1f5e4540d7/Dockerfile#L412-L413
+#=======
+ENV NOVNC_SHA="b403cb92fb8de82d04f305b4f14fa978003890d7" \
+    WEBSOCKIFY_SHA="558a6439f14b0d85a31145541745e25c255d576b"
+RUN  wget -nv -O noVNC.zip "https://github.com/kanaka/noVNC/archive/${NOVNC_SHA}.zip" \
+ && unzip -x noVNC.zip \
+ && rm noVNC.zip  \
+ && mv noVNC-${NOVNC_SHA} noVNC \
+ && wget -nv -O websockify.zip "https://github.com/kanaka/websockify/archive/${WEBSOCKIFY_SHA}.zip" \
+ && unzip -x websockify.zip \
+ && mv websockify-${WEBSOCKIFY_SHA} ./noVNC/utils/websockify \
+ && rm websockify.zip \
+ && ln noVNC/vnc_auto.html noVNC/index.html
+
+#==============================================
+# Download Adb remote screen
+#==============================================
+RUN wget -nv -O asm.zip "http://adakoda.github.io/android-screen-monitor/download/3.0.0/ASM_3_00.zip" \
+ && unzip asm.zip \
+ && rm asm.zip
+
+#================================================
+# noVNC Default Configurations
+# These Configurations can be changed through -e
+#================================================
+ENV DISPLAY=:0 \
+    SCREEN=0 \
+    SCREEN_WIDTH=1600 \
+    SCREEN_HEIGHT=900 \
+    SCREEN_DEPTH=16 \
+    LOCAL_PORT=5900 \
+    TARGET_PORT=6080 \
+    TIMEOUT=1 \
+    VIDEO_PATH=/tmp/video \
+    LOG_PATH=/var/log/supervisor
+
+#=========================
+# Set default variables
+#=========================
+ENV REAL_DEVICE=True
+ENV BROWSER=android
+
+#===============
+# Expose Ports
+#---------------
+# 4723
+#   Appium port
+# 6080
+#   noVNC port
+# 5555
+#   ADB connection port
+#===============
+EXPOSE 4723 6080 5555
+
+#===================
+# Run docker-appium
+#===================
+COPY src /root/src
+COPY supervisord.conf /root/
+RUN chmod -R +x /root/src && chmod +x /root/supervisord.conf
+
+CMD /usr/bin/supervisord --configuration supervisord.conf

release_real.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+    read -p "Task (build|push|all) : " TASK
+else
+    TASK=$1
+fi
+
+if [ -z "$2" ]; then
+    read -p "Release version: " RELEASE
+else
+    RELEASE=$2
+fi
+
+IMAGE="butomo1989/docker-android"
+FILE_NAME=docker/Real_device
+
+image_version="$IMAGE-real-device:$RELEASE"
+image_latest="$IMAGE-real-device:latest"
+
+function build() {
+  echo "[BUILD] Image name: $image_version and $image_latest"
+  echo "[BUILD] Dockerfile: $FILE_NAME"
+  docker build -t $image_version -f $FILE_NAME .
+  docker tag $image_version $image_latest
+}
+
+function push() {
+  echo "[PUSH] Image name: $image_version and $image_latest"
+  docker push $image_version
+  docker push $image_latest
+}
+
+case $TASK in
+    build)
+        build
+    ;;
+    push)
+        push
+    ;;
+    all)
+        build
+        push
+    ;;
+    *)
+        echo "Invalid environment! Valid options: test, build, push, all"
+    ;;
+esac

src/app.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 import subprocess
+import errno
 
 from src import CONFIG_FILE, ROOT, CHROME_DRIVER
 from src import log
@@ -11,6 +12,13 @@ from src import log
 log.init()
 logger = logging.getLogger('app')
 
+def symlink_force(target, link_name):
+    try:
+        os.symlink(target, link_name)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            os.remove(link_name)
+            os.symlink(target, link_name)
 
 def get_or_raise(env: str) -> str:
     """
@@ -71,7 +79,7 @@ def prepare_avd(device: str, avd_name: str):
     skin_dst_path = os.path.join(ANDROID_HOME, 'platforms', 'android-{api}'.format(api=API_LEVEL), 'skins')
     logger.info('Skin destination path: {dst}'.format(dst=skin_dst_path))
     for s in os.listdir(skin_rsc_path):
-        os.symlink(os.path.join(skin_rsc_path, s), os.path.join(skin_dst_path, s))
+        symlink_force(os.path.join(skin_rsc_path, s), os.path.join(skin_dst_path, s))
 
     # Hardware and its skin
     device_name_bash = device.replace(' ', '\ ')
@@ -85,7 +93,7 @@ def prepare_avd(device: str, avd_name: str):
         profile_src_path = os.path.join(ROOT, 'devices', 'profiles', '{profile}.xml'.format(profile=skin_name))
         logger.info('Hardware profile resource path: {rsc}'.format(rsc=profile_src_path))
         logger.info('Hardware profile destination path: {dst}'.format(dst=profile_dst_path))
-        os.symlink(profile_src_path, profile_dst_path)
+        symlink_force(profile_src_path, profile_dst_path)
 
     # Append command
     cmd += ' -d {device} -s {skin}'.format(device=device_name_bash, skin=skin_name)

src/appium.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [ -z $REAL_DEVICE ]; then
+  python3 -m src.app
+else
+  CMD="appium"
+  if [ ! -z "$CONNECT_TO_GRID" ]; then
+    NODE_CONFIG_JSON="/root/src/nodeconfig.json"
+    /root/generate_config.sh $NODE_CONFIG_JSON
+    CMD+=" --nodeconfig $NODE_CONFIG_JSON"
+  fi
+  $CMD
+fi

src/asm.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "$REAL_DEVICE"]; then
+  echo "Container is using android emulator"
+else
+  echo "Starting android screen mirro..."
+  java -jar /root/asm.jar $ANDROID_HOME
+fi

src/record.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+function start() {
+    mkdir -p $VIDEO_PATH
+    name="$(date '+%d_%m_%Y_%H_%M_%S').mp4"
+    echo "Start video recording"
+    ffmpeg -video_size 1599x899 -framerate 15 -f x11grab -i $DISPLAY $VIDEO_PATH/$name -y
+}
+
+function stop() {
+    echo "Stop video recording"
+    kill $(ps -ef | grep [f]fmpeg | awk '{print $2}')
+}
+
+function auto_record() {
+    echo "Auto record: $AUTO_RECORD"
+    sleep 6
+
+    while [ $AUTO_RECORD == "True" ]; do
+        # Check if there is test running
+        no_test=true
+        while $no_test; do
+            task=$(curl -s localhost:4723/wd/hub/sessions | jq -r '.value')
+            if [ "$task" == "" ] || [ "$task" == "[]" ]; then
+                sleep .5
+            else
+                start &
+                no_test=false
+            fi
+        done
+
+        # Check if test is finished
+        while [ $no_test == false ]; do
+            task=$(curl -s localhost:4723/wd/hub/sessions | jq -r '.value')
+            if [ "$task" == "" ] || [ "$task" == "[]" ]; then
+                stop
+                no_test=true
+            else
+                sleep .5
+            fi
+        done
+    done
+
+    echo "Auto recording is disabled!"
+}
+
+$@

src/tests/unit/test_app.py

@@ -10,6 +10,23 @@ from src import app
 class TestApp(TestCase):
     """Unit test class to test other methods in the app."""
 
+    #create symlink
+    @classmethod
+    def test_symlink_correct(self):
+        os.mknod(os.path.join("./","testFile1.txt"))
+        app.symlink_force(os.path.join("./","testFile1.txt"),os.path.join("./","link_testFile1.txt"))
+        os.remove(os.path.join("./","testFile1.txt"))
+        os.remove(os.path.join("./","link_testFile1.txt"))
+
+    #link already exist
+    @classmethod
+    def test_symlink_already_exist(self):
+        os.mknod(os.path.join("./","testFile2.txt"))
+        os.mknod(os.path.join("./","link_testFile2.txt"))
+        app.symlink_force(os.path.join("./","testFile2.txt"),os.path.join("./","link_testFile2.txt"))
+        os.remove(os.path.join("./","testFile2.txt"))
+        os.remove(os.path.join("./","link_testFile2.txt"))
+
     def test_valid_env(self):
         key = 'ENV_1'
         os.environ[key] = 'test'

supervisord.conf

@@ -2,33 +2,56 @@
 nodaemon=true
 logfile=%(ENV_LOG_PATH)s/supervisord.log
 childlogdir=%(ENV_LOG_PATH)s
+priority=1
 
 [program:xvfb]
 command=/usr/bin/Xvfb %(ENV_DISPLAY)s -screen %(ENV_SCREEN)s %(ENV_SCREEN_WIDTH)sx%(ENV_SCREEN_HEIGHT)sx%(ENV_SCREEN_DEPTH)s
 stdout_logfile=%(ENV_LOG_PATH)s/xvfb.stdout.log
 stderr_logfile=%(ENV_LOG_PATH)s/xvfb.stderr.log
+priority=1
 
 [program:openbox]
 command=/usr/bin/openbox-session
 stdout_logfile=%(ENV_LOG_PATH)s/openbox.stdout.log
 stderr_logfile=%(ENV_LOG_PATH)s/openbox.stderr.log
+priority=2
 
 [program:x11vnc]
 command=/usr/bin/x11vnc -display %(ENV_DISPLAY)s -nopw -forever -shared
 stdout_logfile=%(ENV_LOG_PATH)s/x11vnc.stdout.log
 stderr_logfile=%(ENV_LOG_PATH)s/x11vnc.stderr.log
+priority=2
 
 [program:novnc]
 command=./noVNC/utils/launch.sh --vnc localhost:%(ENV_LOCAL_PORT)s --listen %(ENV_TARGET_PORT)s
 stdout_logfile=%(ENV_LOG_PATH)s/novnc.stdout.log
 stderr_logfile=%(ENV_LOG_PATH)s/novnc.stderr.log
+priority=2
 
 [program:port-forward]
 command=./src/port_forward.sh
 autorestart=false
+priority=1
+
+;startsecs to fix "not expected" error. see: https://github.com/Supervisor/supervisor/issues/212#issuecomment-47933372
+[program:android-screen-mirror]
+command=./src/asm.sh
+autorestart=false
+startsecs=0
+stdout_logfile=%(ENV_LOG_PATH)s/android-screen-mirror.stdout.log
+stderr_logfile=%(ENV_LOG_PATH)s/android-screen-mirror.stderr.log
+priority=3
 
 [program:docker-appium]
-command=python3 -m src.app
+command=./src/appium.sh
 autorestart=false
 stdout_logfile=%(ENV_LOG_PATH)s/docker-android.stdout.log
 stderr_logfile=%(ENV_LOG_PATH)s/docker-android.stderr.log
+priority=4
+
+[program:auto-recording]
+command=./src/record.sh auto_record
+autorestart=false
+stdout_logfile=%(ENV_LOG_PATH)s/video-recording.stdout.log
+stderr_logfile=%(ENV_LOG_PATH)s/video-recording.stderr.log
+priority=4

travis.sh

@@ -5,10 +5,15 @@ if [ -z "$TRAVIS_TAG" ]; then
     echo "UNIT TEST ONLY"
     bash release.sh test all all 0.1
 else
-    echo "Log in to docker hub"
+    if [ ! -z "$ANDROID_VERSION" ]; then
-    docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+      echo "Log in to docker hub"
-    echo "[Version: $ANDROID_VERSION] RUN UNIT TEST, BUILD DOCKER IMAGES AND PUSH THOSE TO DOCKER HUB"
+      docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
-    bash release.sh all $ANDROID_VERSION all $TRAVIS_TAG
+      echo "[Version: $ANDROID_VERSION] RUN UNIT TEST, BUILD DOCKER IMAGES AND PUSH THOSE TO DOCKER HUB"
+      bash release.sh all $ANDROID_VERSION all $TRAVIS_TAG
+    elif [ ! -z "$REAL_DEVICE" ]; then
+      echo "[SUPPORT FOR REAL DEVICE: BUILD DOCKER IMAGES AND PUSH THOSE TO DOCKER HUB ]"
+      bash release_real.sh all $TRAVIS_TAG
+    fi
     echo "Log out of docker hub"
     docker logout
 fi