GNOME.org
 

[gjs: 1/2] CI: move the Docker images creation to GitLab

commit c4b508f639163d439b23362a68d0aa1afc227133
Author: Claudio André <claudioandre br gmail com>
Date:   Mon Jul 9 13:24:03 2018 -0300

    CI: move the Docker images creation to GitLab
    
    [skip ci]

 .gitlab-ci.yml                          | 83 ++++++++++++++++++++++++++++++++-
 test/ci-images.sh                       | 45 ++++++++++++++++++
 test/extra/Dockerfile.arm64v8.fedora.29 | 11 +++++
 test/extra/Dockerfile.ppc64le.fedora.29 | 11 +++++
 test/extra/do_basic.sh                  | 77 ++++++++++++++++++++++++++++++
 test/extra/do_docker.sh                 | 39 ++++++++++++++++
 test/extra/do_environment.sh            | 42 +++++++++++++----
 test/extra/do_mozilla.sh                | 53 +++++++++++++++++++++
 test/test-ci.sh                         | 35 +-------------
 9 files changed, 353 insertions(+), 43 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index daf4af67..99c5f48e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -687,19 +687,38 @@ ppc64le:
         exit 0
       fi
 
+    # Get multiarch stuff
+    - |
+      if [[ -n "${TARGET_ARCH}" ]]; then
+        docker run --rm --privileged multiarch/qemu-user-static:register --reset
+        wget 
https://github.com/multiarch/qemu-user-static/releases/download/v2.12.0/x86_64_qemu-${TARGET_ARCH}-static.tar.gz
+      fi
+
     # Build using the Dockerfile
     - |
       if [[ -n "${DOCKERFILE}" ]]; then
         docker build -f "$DOCKERFILE" -t "$CI_REGISTRY_IMAGE:$NAME" .
       fi
 
+    # Where the real magic happens
+    - |
+      if [[ -n "${IMAGE}" ]]; then
+        docker run --name $NAME -v $(pwd):/on-host \
+          -e BASE=$BASE -e OS=$IMAGE -e BUILD_OPTS=$BUILD_OPTS -e DEV=$DEV -e CC=gcc -e STATIC=$STATIC 
$IMAGE \
+          bash -e -c "cd /on-host && test/ci-images.sh BUILD_MOZ"
+        docker commit $NAME "$CI_REGISTRY_IMAGE:$NAME"
+      fi
+
     # Prepare to publish
     - docker tag "$CI_REGISTRY_IMAGE:$NAME" "$CI_REGISTRY_IMAGE:job-"$CI_JOB_ID"_$NAME"
     - docker images
     - docker login ${CI_REGISTRY} -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD}
 
-    # Publish
-    - docker push $CI_REGISTRY_IMAGE
+    # Publish (if running on a schedule)
+    - |
+      if [[ "${CI_PIPELINE_SOURCE}" == "schedule" ]]; then
+        docker push "$CI_REGISTRY_IMAGE"
+      fi
 
     # Done
     - *CI_footer
@@ -711,3 +730,63 @@ fedora.static-analysis:
     DOCKER_DRIVER: overlay
     DOCKERFILE: "test/extra/Dockerfile.fedora.static-analysis"
     NAME: "fedora.static-analysis"
+
+SM52-gcc.fedora-dev:
+  <<: *create_docker_image
+  variables:
+    BASE: "fedora"
+    DEV: "devel"
+    DOCKER_DRIVER: overlay
+    IMAGE: "fedora:rawhide"
+    NAME: "SM52-gcc.fedora-dev"
+
+SM52-debug-gcc.fedora-dev:
+  <<: *create_docker_image
+  variables:
+    BASE: "fedora"
+    BUILD_OPTS: "--enable-debug"
+    DEV: "devel"
+    DOCKER_DRIVER: overlay
+    IMAGE: "fedora:rawhide"
+    NAME: "SM52-debug-gcc.fedora-dev"
+
+SM52-gcc.fedora-dev.aarch64:
+  <<: *create_docker_image
+  variables:
+    BASE: "fedora"
+    DEV: "devel"
+    DOCKER_DRIVER: overlay
+    DOCKERFILE: "test/extra/Dockerfile.arm64v8.fedora.29"
+    IMAGE: "${CI_REGISTRY_IMAGE}:SM52-gcc.fedora-dev.aarch64"
+    NAME: "SM52-gcc.fedora-dev.aarch64"
+    STATIC: "qemu"
+    TARGET_ARCH: "aarch64"
+
+SM52-gcc.fedora-dev.ppc64le:
+  <<: *create_docker_image
+  variables:
+    BASE: "fedora"
+    DEV: "devel"
+    DOCKER_DRIVER: overlay
+    DOCKERFILE: "test/extra/Dockerfile.ppc64le.fedora.29"
+    IMAGE: "${CI_REGISTRY_IMAGE}:SM52-gcc.fedora-dev.ppc64le"
+    NAME: "SM52-gcc.fedora-dev.ppc64le"
+    STATIC: "qemu"
+    TARGET_ARCH: "ppc64le"
+
+SM52-gcc.ubuntu-lts:
+  <<: *create_docker_image
+  variables:
+    BASE: "debian"
+    DOCKER_DRIVER: overlay
+    IMAGE: "ubuntu:18.04"
+    NAME: "SM52-gcc.ubuntu-lts"
+
+SM52-gcc.ubuntu-dev:
+  <<: *create_docker_image
+  variables:
+    BASE: "debian"
+    DEV: "devel"
+    DOCKER_DRIVER: overlay
+    IMAGE: "ubuntu:devel"
+    NAME: "SM52-gcc.ubuntu-dev"
diff --git a/test/ci-images.sh b/test/ci-images.sh
new file mode 100755
index 00000000..6c685e8a
--- /dev/null
+++ b/test/ci-images.sh
@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+function do_Set_Env(){
+    echo
+    echo '-- Set Environment --'
+
+    #Save cache on host (outside the image), if linked
+    mkdir -p /on-host/.cache
+    export XDG_CACHE_HOME=/on-host/.cache
+
+    export JHBUILD_RUN_AS_ROOT=1
+    export SHELL=/bin/bash
+    PATH=$PATH:~/.local/bin
+
+    if [[ -z "${DISPLAY}" ]]; then
+        export DISPLAY=":0"
+    fi
+
+    echo '-- Done --'
+}
+
+# ----------- Run the Tests -----------
+cd /on-host
+
+if [[ -n "${BUILD_OPTS}" ]]; then
+    extra_opts="($BUILD_OPTS)"
+fi
+
+if [[ -n "${STATIC}" ]]; then
+    extra_opts="$extra_opts  ($STATIC)"
+fi
+
+source test/extra/do_environment.sh
+
+# Show some environment info
+do_Print_Labels  'ENVIRONMENT'
+echo "Running on: $BASE $OS"
+echo "Doing: $1 $extra_opts"
+
+source test/extra/do_basic.sh
+source test/extra/do_mozilla.sh
+source test/extra/do_docker.sh
+
+# Releases stuff and finishes
+do_Done
diff --git a/test/extra/Dockerfile.arm64v8.fedora.29 b/test/extra/Dockerfile.arm64v8.fedora.29
new file mode 100644
index 00000000..92a6b03e
--- /dev/null
+++ b/test/extra/Dockerfile.arm64v8.fedora.29
@@ -0,0 +1,11 @@
+FROM arm64v8/fedora:rawhide
+MAINTAINER Claudio André (c) 2018 V1.0
+
+LABEL architecture="aarch64"
+LABEL version="1.0"
+LABEL description="Multiarch docker image to run CI for GNOME GJS."
+
+ADD x86_64_qemu-aarch64-static.tar.gz /usr/bin
+
+CMD ["/bin/bash"]
+
diff --git a/test/extra/Dockerfile.ppc64le.fedora.29 b/test/extra/Dockerfile.ppc64le.fedora.29
new file mode 100644
index 00000000..400d81c3
--- /dev/null
+++ b/test/extra/Dockerfile.ppc64le.fedora.29
@@ -0,0 +1,11 @@
+FROM ppc64le/fedora:rawhide
+MAINTAINER Claudio André (c) 2018 V1.0
+
+LABEL architecture="ppc64le"
+LABEL version="1.0"
+LABEL description="Multiarch docker image to run CI for GNOME GJS."
+
+ADD x86_64_qemu-ppc64le-static.tar.gz /usr/bin
+
+CMD ["/bin/bash"]
+
diff --git a/test/extra/do_basic.sh b/test/extra/do_basic.sh
new file mode 100755
index 00000000..de2ea820
--- /dev/null
+++ b/test/extra/do_basic.sh
@@ -0,0 +1,77 @@
+#!/bin/bash -e
+
+function do_Install_Base_Dependencies(){
+    echo
+    echo '-- Installing Base Dependencies --'
+
+    if [[ $BASE == "debian" ]]; then
+        apt-get update -qq
+
+        # Base dependencies
+        apt-get -y -qq install build-essential git clang patch bison flex \
+                               python-dev python3-dev \
+                               autotools-dev autoconf gettext pkgconf autopoint yelp-tools \
+                               docbook docbook-xsl libtext-csv-perl \
+                               zlib1g-dev libdbus-glib-1-dev \
+                               libtool libicu-dev libnspr4-dev \
+                               policykit-1 > /dev/null
+
+    elif [[ $BASE == "fedora" ]]; then
+        if [[ $STATIC == *"qemu"* ]]; then
+            dnf -y -q --nogpgcheck upgrade
+        else
+            dnf -y -q upgrade
+        fi
+
+        # Base dependencies
+        dnf -y -q install @c-development @development-tools clang redhat-rpm-config gnome-common 
python-devel \
+                          pygobject2 dbus-python perl-Text-CSV perl-XML-Parser gettext-devel gtk-doc 
ninja-build \
+                          zlib-devel libffi-devel \
+                          libtool libicu-devel nspr-devel
+    else
+        echo
+        echo '-- Error: invalid BASE code --'
+        exit 1
+    fi
+}
+
+function do_Install_Dependencies(){
+    echo
+    echo '-- Installing Dependencies --'
+
+    if [[ $BASE == "debian" ]]; then
+        # Testing dependencies
+        apt-get -y -qq install libgtk-3-dev gir1.2-gtk-3.0 xvfb gnome-desktop-testing dbus-x11 dbus \
+                               libedit-dev libgl1-mesa-dev lcov > /dev/null
+
+    elif [[ $BASE == "fedora" ]]; then
+        # Testing dependencies
+        dnf -y -q install gtk3 gtk3-devel gobject-introspection Xvfb gnome-desktop-testing dbus-x11 dbus \
+                          cairo intltool libxslt bison nspr zlib python3-devel dbus-glib libicu libffi pcre 
libxml2 libxslt libtool flex \
+                          cairo-devel zlib-devel libffi-devel pcre-devel libxml2-devel libxslt-devel \
+                          libedit-devel libasan libubsan libtsan compiler-rt lcov mesa-libGL-devel
+
+        if [[ $DEV == "devel" ]]; then
+            dnf -y -q install time
+        fi
+    fi
+}
+
+function do_Install_Extras(){
+    echo
+    echo '-- Installing Extra Dependencies --'
+
+    if [[ $BASE == "debian" ]]; then
+        # Distros development versions of needed libraries
+        apt-get -y -qq install libgirepository1.0-dev > /dev/null
+
+    elif [[ $BASE == "fedora" ]]; then
+        # Distros development versions of needed libraries
+        dnf -y -q install gobject-introspection-devel
+
+        if [[ $STATIC != *"qemu"* ]]; then
+            # Distros debug info of needed libraries
+            dnf -y debuginfo-install glib2-devel gobject-introspection-devel gtk3-devel expat fontconfig 
cairo glibc
+        fi
+    fi
+}
diff --git a/test/extra/do_docker.sh b/test/extra/do_docker.sh
new file mode 100755
index 00000000..1566e59d
--- /dev/null
+++ b/test/extra/do_docker.sh
@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+function do_Shrink_Image(){
+    echo
+    echo '-- Cleaning image --'
+    PATH=$PATH:~/.local/bin
+    rm -rf ~/jhbuild/install/lib/libjs_static.ajs
+
+    if [[ $BASE == "ubuntu" ]]; then
+        apt-get -y clean
+        rm -rf /var/lib/apt/lists/*
+
+    elif [[ $BASE == "fedora" ]]; then
+        dnf -y clean all
+    fi
+
+    echo '-- Done --'
+}
+
+if [[ $1 == "BUILD_MOZ" ]]; then
+    do_Install_Base_Dependencies
+    do_Install_Dependencies
+    do_Set_Env
+
+    if [[ $DEV == "devel" ]]; then
+        do_Install_Extras
+    fi
+    do_Show_Info
+    do_Get_JHBuild
+    do_Build_JHBuild
+    do_Build_Mozilla
+
+    # Build JHBuild to create a docker image ready to go
+    jhbuild build m4-common
+
+    do_Shrink_Image
+fi
+# Clear the environment
+unset BUILD_OPTS
diff --git a/test/extra/do_environment.sh b/test/extra/do_environment.sh
index 51a5cd27..f34c5c5f 100755
--- a/test/extra/do_environment.sh
+++ b/test/extra/do_environment.sh
@@ -1,8 +1,7 @@
 #!/bin/bash -e
 
 function do_Get_JHBuild(){
-    echo
-    echo '-- Download JHBuild --'
+    do_Print_Labels 'Download JHBuild'
 
     if [[ -d /jhbuild ]]; then
         # For a clean build, update and rebuild jhbuild. And avoid git pull.
@@ -18,8 +17,7 @@ function do_Get_JHBuild(){
 }
 
 function do_Configure_JHBuild(){
-    echo
-    echo '-- Set JHBuild Configuration --'
+    do_Print_Labels 'Set JHBuild Configuration'
 
     mkdir -p ~/.config
 
@@ -32,8 +30,7 @@ EOFILE
 }
 
 function do_Configure_MainBuild(){
-    echo
-    echo '-- Set Main JHBuild Configuration --'
+    do_Print_Labels 'Set Main JHBuild Configuration'
 
     mkdir -p ~/.config
     autogenargs="--enable-compile-warnings=yes --with-xvfb-tests"
@@ -54,9 +51,16 @@ EOFILE
     echo '-- Done --'
 }
 
+function do_Build_Package_Dependencies(){
+    do_Print_Labels "Building Dependencies for $1"
+    jhbuild list "$1"
+
+    # Build package dependencies
+    jhbuild build $(jhbuild list "$1" | sed '$d')
+}
+
 function do_Build_JHBuild(){
-    echo
-    echo '-- Building JHBuild --'
+    do_Print_Labels 'Building JHBuild'
 
     # Build JHBuild
     cd /jhbuild
@@ -74,6 +78,28 @@ function do_Build_JHBuild(){
     cd -
 }
 
+function do_Print_Labels(){
+
+    if [[ -n "${1}" ]]; then
+        label_len=${#1}
+        span=$(((54 - $label_len) / 2))
+
+        echo
+        echo "= ======================================================== ="
+        printf "%s %${span}s %s %${span}s %s\n" "=" "" "$1" "" "="
+        echo "= ======================================================== ="
+    else
+        echo "= ========================= Done ========================= ="
+        echo
+    fi
+}
+
+function do_Done(){
+
+    # Done. De-initializes whatever is needed
+    do_Print_Labels  'FINISHED'
+}
+
 function do_Show_Info(){
 
     local compiler=gcc
diff --git a/test/extra/do_mozilla.sh b/test/extra/do_mozilla.sh
new file mode 100755
index 00000000..847060f9
--- /dev/null
+++ b/test/extra/do_mozilla.sh
@@ -0,0 +1,53 @@
+#!/bin/bash -e
+
+function do_Configure_MozBuild(){
+    echo
+    echo '-- Set JHBuild Configuration --'
+
+    mkdir -p ~/.config
+    autogenargs=""
+
+    if [[ -n "${BUILD_OPTS}" ]]; then
+        autogenargs="$autogenargs $BUILD_OPTS"
+    fi
+
+    cat <<EOFILE >> ~/.config/jhbuildrc
+module_autogenargs['mozjs52'] = "$autogenargs"
+module_makeargs['mozjs52'] = '-s'
+EOFILE
+
+    echo '-- Done --'
+}
+
+function do_Build_Mozilla_jhbuild(){
+    echo
+    echo '-- Building Mozilla SpiderMonkey --'
+
+    # Configure the Mozilla build
+    do_Configure_MozBuild
+
+    # Build Mozilla Stuff
+    jhbuild build mozjs52
+}
+
+function do_Build_Mozilla(){
+    echo
+    echo '-- Building Mozilla SpiderMonkey --'
+
+    if [[ $STATIC == "moz60" ]]; then
+        git clone --depth 1 https://github.com/ptomato/mozjs.git -b mozjs60 /on-host/spider
+        cd /on-host/spider
+    else
+        git clone --depth 1 https://github.com/ptomato/mozjs.git -b mozjs52 /on-host/spider
+        cd /on-host/spider
+    fi
+
+    mkdir -p _build
+    cd _build
+
+    ../js/src/configure --prefix=/root/jhbuild/install --enable-posix-nspr-emulation --with-system-zlib 
--with-intl-api --disable-jemalloc AUTOCONF=autoconf ${BUILD_OPTS}
+    make -sj4
+    make install
+
+    cd -
+}
diff --git a/test/test-ci.sh b/test/test-ci.sh
index 209b9953..d3b1ecd9 100755
--- a/test/test-ci.sh
+++ b/test/test-ci.sh
@@ -1,21 +1,5 @@
 #!/bin/bash -e
 
-function do_Print_Labels(){
-
-    if [[ -n "${1}" ]]; then
-        label_len=${#1}
-        span=$(((54 - $label_len) / 2))
-
-        echo
-        echo "= ======================================================== ="
-        printf "%s %${span}s %s %${span}s %s\n" "=" "" "$1" "" "="
-        echo "= ======================================================== ="
-    else
-        echo "= ========================= Done ========================= ="
-        echo
-    fi
-}
-
 function do_Set_Env(){
 
     do_Print_Labels 'Set Environment '
@@ -43,21 +27,6 @@ function do_Set_Env(){
     do_Print_Labels
 }
 
-function do_Done(){
-
-    # Done. De-initializes whatever is needed
-    do_Print_Labels  'FINISHED'
-}
-
-function do_Build_Package_Dependencies(){
-
-    do_Print_Labels "Building Dependencies for $1"
-    jhbuild list "$1"
-
-    # Build package dependencies
-    jhbuild build $(jhbuild list "$1" | sed '$d')
-}
-
 function do_Get_Upstream_Master(){
 
     if [[ "$CI_PROJECT_PATH_SLUG" == "gnome-gjs" && \
@@ -147,6 +116,8 @@ if [[ -n "${TEST}" ]]; then
     extra_opts="($TEST)"
 fi
 
+source test/extra/do_environment.sh
+
 # Show some environment info
 do_Print_Labels  'ENVIRONMENT'
 echo "Running on: $BASE $OS"
@@ -154,8 +125,6 @@ echo "Job: $TASK_ID"
 echo "Build options: $BUILD_OPTS"
 echo "Doing: $1 $extra_opts"
 
-source test/extra/do_environment.sh
-
 do_Create_Artifacts_Folder "$1"
 do_Get_Commit_Message
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]