replace travis with circle-ci, cleanup of build steps (#639)

Travis is getting slower and was already hard to debug. This commit: - moves from travis to Circle-CI - has a ci-scripts/README.md which explains what is contained there - puts all build and CI scripts into "ci_scripts" directory - allows to run full ci runs locally - trims down the base Docker image where tests/docs/wheels are run from 4.4GB to 1.7GB - fixes #628 Using circle-ci also means that we can probably integrate Mac-OS build steps more easily. I am sure that my initial circle-ci config (helped by kind hints from @dignifiedquire ) can be improved.
2025-10-04 10:19:16 +02:00 · 2019-04-01 09:45:56 +02:00 · 2019-04-01 09:45:56 +02:00 · 515e5ebba7
commit 515e5ebba7
parent 7f3bf1f528
34 changed files with 608 additions and 309 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -0,0 +1 @@
+../ci_scripts/config.yml
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,11 @@
 # ignore backup files
 *~

+# ignore tox and docker stuff
+.docker-*
+wheelhouse
+workspace
+
 # ignore build temporary files
 *.o
 .deps
--- a/.scripts/buildwheel.sh
+++ b/.scripts/buildwheel.sh
@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-docker pull deltachat/wheel 
-rm -rf python/wheelhouse/*
-cd $TRAVIS_BUILD_DIR
-docker run --rm -it -v $(pwd):/io deltachat/wheel /io/python/wheelbuilder/build-wheels.sh
-
-# create an index at the community "devpi" python packaging site
-# and push both binary wheel packages and the source package to
-# the https://m.devpi.net/dc/BRANCHNAME index 
-devpi use https://m.devpi.net
-devpi login dc --password $DEVPI_LOGIN
-
-devpi use dc/$BRANCH || {
-    devpi index -c $BRANCH 
-    devpi use dc/$BRANCH
-}
-devpi index $BRANCH bases=/root/pypi
-
-devpi upload --from-dir python/wheelhouse
-
-cd python
-devpi upload 
--- a/.scripts/deploy.sh
+++ b/.scripts/deploy.sh
@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -e
-set -u
-set -x
-set -v
-
-
-#Only attempt to deploy if we know the ssh key secrets, username and server
-if test -z ${encrypted_49475b8073e9_key:+decryp_key} ; then exit 0; fi
-if test -z ${encrypted_49475b8073e9_iv:+decrypt_iv} ; then exit 0; fi
-if test -z ${DEPLOY_USER:+username} ; then exit 0; fi
-if test -z ${DEPLOY_SERVER:+server} ; then exit 0; fi
-
-# Prepare the ssh homedir.
-#
-# Decrypting the ssh private key for deploying to the server.
-# See https://docs.travis-ci.com/user/encrypting-files/ for details.
-mkdir -p -m 0700 ~/.ssh
-openssl aes-256-cbc \
-  -K $encrypted_49475b8073e9_key \
-  -iv $encrypted_49475b8073e9_iv \
-  -in $TRAVIS_BUILD_DIR/.credentials/delta.id_rsa.enc \
-  -out ~/.ssh/id_rsa -d
-chmod 600 ~/.ssh/id_rsa
-printf "Host *\n" >> ~/.ssh/config
-printf " %sAuthentication no\n" ChallengeResponse Password KbdInteractive >> ~/.ssh/config
-
-
-# Perform the actual deploy to py.delta.chat
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  $TRAVIS_BUILD_DIR/python/doc/_build/html/ \
-  ${DEPLOY_USER}@${DEPLOY_SERVER}:build/${TRAVIS_BRANCH/\//_}
-
-# Perform the actual deploy to c.delta.chat
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  $TRAVIS_BUILD_DIR/docs/html/ \
-  ${DEPLOY_USER}@${DEPLOY_SERVER}:build-c/${TRAVIS_BRANCH/\//_}
--- a/.travis.yml
+++ b/.travis.yml
@ -1,70 +0,0 @@
-sudo: required
-language: c
-env:
-  global:
-    - DEPLOY_USER=delta DEPLOY_SERVER="py.delta.chat"
-    - secure: kxF/4pnq6jnhc/9BCtO8kQnoFg6w3/SMfy4zE54pqCzJEHKmOTVV0B+XX7epwYqOMI3aDDprVp+3+qi9+soz8ISN2Sedu1ZRje2ILv/OhHJUVkMLgAmDzzPBhlvlVXA18baIohxjJLGFHw2ZLz9aKHZENcM33ujs/4bSdODuWw6Re8C29aolh6+gvDlb/TmyB27GZoFa3tyc0EU0IhEYX4sPMkkFJVefvTPeKlQzKwJ0InfGMoWhlpr8CqFLsPHbcbDGrkoWXmmMOQny49itsr0AQoe9jpRnHAejrunOKlEnB5q+ok4p/1AQb5K7rZ0m93jbgyjT8Gz2GOeM49xd5M75ZzaLJXs6EkaWg7Y8kP7d7QpkXmKeQK+VjP1n7k2u+hGl9v89mk5aDvhceBBr4v0CjfLZKL3INd2/Z8CBggMyjpGhri5BCNOHxlFycqnsGv0ssz+BSElGQ8KeKtMVfqxkdikPTSpRCX+FT7DfZbM8KiDk0ezI3dEDRBMSZtQ4fngxdasC5FfrH8Y6g6LxAkByfwpGij6Rio0A7ICjlg8b0sJ7iEiFHy5lfkMSm+yoSEzyjFQb/D9JxnsnCTfqL+qr8j3IGGswEnDP9iRkw156gHneFsOMYW6wOtci7ZzYxdxTCGRt9y2HQiQQn8VgI9fLI77/ijDQmt13s1StikE=
-  matrix:
-    - MESONARGS="" DOCS=t TESTS=t WHEEL=t
-    - MESONARGS="-Dmonolith=true" TESTS=t
-    - MESONARGS="--default-library=static"
-    - MESONARGS="--wrap-mode=forcefallback --default-library=static"
-addons:
-  apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    packages:
-    - python3-pip
-    - python3.5
-    - g++-7
-    - libssl-dev
-    - libsqlite3-dev
-    - libbz2-dev
-    - zlib1g-dev
-    - python3.5-dev
-    - python-software-properties
-    - doxygen
-
-services:
-  - docker
-
-install:
-  - export CC=gcc-7
-  - export CXX=g++-7
-  - sudo ln -sf /usr/bin/python3.5 /usr/bin/python3
-  - sudo pip3 install meson
-  - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-  - unzip ninja-linux.zip
-  - sudo cp ninja /usr/bin
-  - wget http://http.debian.net/debian/pool/main/c/cyrus-sasl2/cyrus-sasl2_2.1.27~101-g0780600+dfsg.orig.tar.xz
-  - tar xfv cyrus-sasl2_2.1.27~101-g0780600+dfsg.orig.tar.xz
-  - pushd cyrus-sasl2-2.1.27~101-g0780600+dfsg.orig && ./autogen.sh && make && sudo
-      make install && popd
-
-script:
-  - export BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}
-  - echo "BRANCH=$BRANCH"
-  - doxygen --version
-  - mkdir -p builddir && pushd builddir
-  - meson $MESONARGS && ninja -v && sudo ninja install
-  - export LD_LIBRARY_PATH=/usr/local/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
-  - sudo ldconfig -v
-  - popd && pushd docs
-  - if [ -n "$DOCS" ]; then doxygen; fi
-  - popd && pushd python
-  - virtualenv -p /usr/bin/python3.5 venv
-  - source venv/bin/activate
-  - pip install tox devpi-client
-  - if [ -e /usr/local/lib/x86_64-linux-gnu/libdeltachat.so ]; then ldd /usr/local/lib/x86_64-linux-gnu/libdeltachat.so;
-      fi
-  - if [ -n "$TESTS" ]; then tox; fi
-  - if [ -n "$DOCS" ]; then tox -e doc; fi
-  - if [ -n "$WHEEL" ]; then bash $TRAVIS_BUILD_DIR/.scripts/buildwheel.sh ; fi
-  - popd
-deploy:
-  provider: script
-  skip_cleanup: true
-  script: bash $TRAVIS_BUILD_DIR/.scripts/deploy.sh
-  on:
-    all_branches: true
-    condition: $DOCS
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
 # Delta Chat Core Library

-[![Build Status](https://travis-ci.org/deltachat/deltachat-core.svg?branch=master)](https://travis-ci.org/deltachat/deltachat-core)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core.svg?style=svg)](https://circleci.com/gh/deltachat/deltachat-core)

 The _Delta Chat Core Library_ is written in cross-platform **C**,
 documented at <https://c.delta.chat>.  
@ -9,6 +9,9 @@ The ``deltachat`` Python bindings can be found in the
 [python subdirectory](https://github.com/deltachat/deltachat-core/tree/master/python)
 and are documented at <https://py.delta.chat>.

+In the [ci_scripts directory](https://github.com/deltachat/deltachat-core/tree/master/ci_scripts/README.md)
+you'll find docker- and library-building scripts. 
+
 ## binary/distribution packages  (work-in-progress)

 There are work-in-progress efforts for creating (binary) packages which
@ -16,6 +19,8 @@ do not require that you build the library manually:

 - [libdeltachat-core-git archlinux package](https://aur.archlinux.org/packages/libdeltachat-core-git/>)

+- [python-wheel packaging](https://m.devpi.net/dc/master)
+
 - [Debian packaging](https://github.com/deltachat/deltachat-core/issues/299)

 - [Windows building](https://github.com/deltachat/deltachat-core/issues/306)
--- a/ci_scripts/README.md
+++ b/ci_scripts/README.md
@ -0,0 +1,52 @@
+
+# Continuous Integration Scripts for Delta Chat
+
+Continuous Integration is run through CircleCI
+but is largely independent of it. 
+
+
+## Generating docker containers for performing build step work 
+
+All tests, docs and wheel building is run in docker containers:
+
+- **coredeps/Dockerfile** specifies an image that contains all 
+  of Delta Chat's core dependencies as linkable libraries. 
+  It also serves to run python tests and build wheels 
+  (binary packages for Python). 
+
+- **doxygen/Dockerfile** specifies an image that contains
+  the doxygen tool which is used to generate C-docs. 
+
+To run tests locally you can pull existing images from "docker.io",
+the hub for sharing Docker images::
+
+    docker pull deltachat/coredeps
+    docker pull deltachat/doxygen 
+
+or you can build the docker images yourself locally
+to avoid the relatively large download:: 
+ 
+    cd ci_scripts  # where all CI things are 
+    docker build -t deltachat/coredeps docker-coredeps
+    docker build -t deltachat/doxygen docker-doxygen 
+
+## ci_run.sh (main entrypoint called by circle-ci)
+
+Once you have the docker images available 
+you can run python testing, documentation generation 
+and building binary wheels:: 
+
+    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
+    
+## ci_upload.sh (uploading artifacts on success) 
+
+- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+
+- python wheels to `https://m.devpi.net/dc/<BRANCH>`
+  so that you install fully self-contained wheels like this:
+  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
+
+
+
--- a/ci_scripts/ci_run.sh
+++ b/ci_scripts/ci_run.sh
@ -0,0 +1,20 @@
+# perform CI jobs on PRs and after merges to master.
+# triggered from .circleci/config.yml
+
+set -e -x
+
+export BRANCH=${CIRCLE_BRANCH:-test7}
+    
+# run doxygen on c-source (needed by later doc-generation steps).
+# XXX modifies the host filesystem docs/xml and docs/html directories
+# XXX which you can then only remove with sudo as they belong to root
+
+if [ -n "$DOCS" ] ; then 
+    docker run --rm -it -v $PWD:/mnt -w /mnt/docs deltachat/doxygen doxygen
+fi
+
+# run everything else inside docker (TESTS, DOCS, WHEELS) 
+docker run -e BRANCH -e MESONARGS -e TESTS -e DOCS \
+           --rm -it -v $(pwd):/mnt -w /mnt \
+           deltachat/coredeps ci_scripts/run_all.sh
+
--- a/ci_scripts/ci_upload.sh
+++ b/ci_scripts/ci_upload.sh
@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 1
+fi
+
+set -xe
+
+DOXYDOCDIR=${1:?directory where doxygen docs to be found}
+PYDOCDIR=${2:?directory with python docs}
+WHEELHOUSEDIR=${3:?directory with pre-built wheels}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@py.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+devpi use dc/$BRANCH || {
+    devpi index -c $BRANCH 
+    devpi use dc/$BRANCH
+}
+devpi index $BRANCH bases=/root/pypi
+devpi upload deltachat*.whl
+
+popd
--- a/ci_scripts/config.yml
+++ b/ci_scripts/config.yml
@ -0,0 +1,90 @@
+
+version: 2
+jobs:
+  build_test_docs_wheel:
+    machine: True
+    steps:
+      - checkout
+      - run: docker pull deltachat/doxygen 
+      - run: docker pull deltachat/coredeps
+      - run: 
+          command: ci_scripts/ci_run.sh
+          environment:
+            MESONARGS: 
+            DOCS: 1
+            TESTS: 1
+
+      - run: 
+           name: copying C docs, python docs and wheels to workspace 
+           command: |
+             mkdir -p workspace/python
+             cp -av docs workspace/c-docs
+             cp -av python/.docker-tox/wheelhouse workspace/
+             cp -av python/doc/_build/ workspace/py-docs
+
+      - persist_to_workspace: 
+          root: workspace 
+          paths: 
+            - c-docs
+            - py-docs
+            - wheelhouse
+
+  upload_docs_wheels:
+    machine: True
+    steps:
+      - run:
+          command: |
+            echo 'export MESONARGS=""' >> $BASH_ENV 
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: ls -laR workspace 
+      - run: ci_scripts/ci_upload.sh workspace/c-docs workspace/py-docs workspace/wheelhouse
+
+  build_monolith_tests:
+    machine: True
+    steps:
+      - checkout
+      - run: docker pull deltachat/coredeps
+      - run:
+          command: |
+            echo 'export MESONARGS="-Dmonolith=true"' >> $BASH_ENV
+            echo 'export TESTS=1' >> $BASH_ENV 
+      - run: ci_scripts/ci_run.sh
+
+  build_static:
+    machine: True
+    steps:
+      - checkout
+      - run: docker pull deltachat/coredeps
+      - run:
+          command: |
+            echo 'export MESONARGS="--default-library=static"' >> $BASH_ENV
+      - run: ci_scripts/ci_run.sh
+
+  build_forcefallback: 
+    machine: True
+    steps:
+      - checkout
+      - run: docker pull deltachat/coredeps
+      - run:
+          command: |
+            echo 'export MESONARGS="--wrap-mode=forcefallback --default-library=static"' >> $BASH_ENV
+      - run: ci_scripts/ci_run.sh
+
+workflows:
+  version: 2
+  build_all:
+    jobs:
+      - build_test_docs_wheel
+      - build_monolith_tests
+      - build_static
+      - build_forcefallback
+      - upload_docs_wheels:
+          requires:
+              - build_test_docs_wheel
+              - build_monolith_tests
+              - build_static
+              - build_forcefallback
+        
+
--- a/ci_scripts/docker-coredeps/Dockerfile
+++ b/ci_scripts/docker-coredeps/Dockerfile
@ -0,0 +1,44 @@
+FROM quay.io/pypa/manylinux1_x86_64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install ninja
+ADD deps/build_ninja.sh /scripts/build_ninja.sh
+RUN mkdir tmp1 && cd tmp1 && bash /scripts/build_ninja.sh && cd .. && rm -r tmp1
+
+# Install a recent zlib, needed by libetpan
+ADD deps/build_zlib.sh /builder/build_zlib.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_zlib.sh && cd .. && rm -r tmp1
+
+# Install a recent Perl, needed to install OpenSSL
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+# Install OpenSSL
+ADD deps/build_openssl.sh /builder/build_openssl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
+
+# Install cyrus-sasl
+ADD deps/build_sasl.sh /builder/build_sasl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_sasl.sh && cd .. && rm -r tmp1
+
+# Install libetpan
+ADD deps/build_libetpan.sh /builder/build_libetpan.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_libetpan.sh && cd .. && rm -r tmp1
+
+# Install python tools (meson, tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust nightly 
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
+
+# Install RPGP from current github (beware, will be static in the docker image) 
+ADD deps/build_rpgp.sh /builder/build_rpgp.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rpgp.sh && cd .. && rm -r tmp1
--- a/ci_scripts/docker-coredeps/deps/build_libetpan.sh
+++ b/ci_scripts/docker-coredeps/deps/build_libetpan.sh
@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e -x
+
+ETPAN_VERSION=1.9.1
+ETPAN_SHA256=f5e354ccf1014c6ee313ade1009b8a82f28043d2504655e388bb4c1328700fcd
+curl -L -o libetpan-${ETPAN_VERSION}.tar.gz \
+    https://github.com/dinhviethoa/libetpan/archive/${ETPAN_VERSION}.tar.gz
+echo "${ETPAN_SHA256}  libetpan-${ETPAN_VERSION}.tar.gz" | sha256sum -c -
+tar xzf libetpan-${ETPAN_VERSION}.tar.gz
+cd libetpan-${ETPAN_VERSION} 
+./autogen.sh && \
+./configure --enable-ipv6 \
+            --enable-iconv --disable-db \
+            --with-openssl --with-sasl --with-zlib \
+            --without-curl --without-expat
+make
+make install
+ldconfig -v | grep -i etpan 
--- a/ci_scripts/docker-coredeps/deps/build_ninja.sh
+++ b/ci_scripts/docker-coredeps/deps/build_ninja.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e -x
+
+NINJA_VERSION=v1.8.2
+NINJA_SHA256=d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07
+
+curl -L -o ninja-linux-$NINJA_VERSION.zip \
+     https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip  
+
+echo "${NINJA_SHA256}  ninja-linux-${NINJA_VERSION}.zip" | sha256sum -c -  
+unzip ninja-linux-${NINJA_VERSION}.zip 
+mv ninja /usr/bin/ninja 
+
--- a/ci_scripts/docker-coredeps/deps/build_openssl.sh
+++ b/ci_scripts/docker-coredeps/deps/build_openssl.sh
@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 
--- a/ci_scripts/docker-coredeps/deps/build_perl.sh
+++ b/ci_scripts/docker-coredeps/deps/build_perl.sh
@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.28.0
+PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION} 
+
+./Configure -de
+make
+make install
--- a/ci_scripts/docker-coredeps/deps/build_python.sh
+++ b/ci_scripts/docker-coredeps/deps/build_python.sh
@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install meson tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/meson 
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd
--- a/ci_scripts/docker-coredeps/deps/build_rpgp.sh
+++ b/ci_scripts/docker-coredeps/deps/build_rpgp.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e -x 
+
+# Install RPGP from github 
+# XXX built a particular version like the other dep scripts do
+
+export PATH=$PATH:$HOME/.cargo/bin
+git clone https://github.com/rpgp/rpgp.git 
+cd rpgp/pgp-ffi 
+make install 
+
+# after we installed the RPGP lib we don't need Rust anymore
+# rm -rf /root/.cargo /root.rustup
--- a/ci_scripts/docker-coredeps/deps/build_rust.sh
+++ b/ci_scripts/docker-coredeps/deps/build_rust.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x 
+
+# Install Rust 
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-03-23-x86_64-unknown-linux-gnu/share/
--- a/ci_scripts/docker-coredeps/deps/build_sasl.sh
+++ b/ci_scripts/docker-coredeps/deps/build_sasl.sh
@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e -x
+
+SASL_VERSION=2.1.27
+SASL_SHA256=26866b1549b00ffd020f188a43c258017fa1c382b3ddadd8201536f72efb05d5
+
+curl -O https://www.cyrusimap.org/releases/cyrus-sasl-${SASL_VERSION}.tar.gz
+echo "${SASL_SHA256}  cyrus-sasl-${SASL_VERSION}.tar.gz" | sha256sum -c -
+tar zxf cyrus-sasl-${SASL_VERSION}.tar.gz
+
+cd cyrus-sasl-${SASL_VERSION} 
+
+./configure --disable-silent-rules \
+            --disable-cmulocal \
+            --disable-sample \
+            --disable-obsolete_cram_attr \
+            --disable-obsolete_digest_attr \
+            --disable-staticdlopen \
+            --disable-java \
+            --disable-alwaystrue \
+            --enable-checkapop \
+            --enable-cram \
+            --enable-digest \
+            --enable-scram \
+            --disable-otp \
+            --disable-srp \
+            --disable-srp-setpass \
+            --disable-krb4 \
+            --disable-gssapi \
+            --disable-gss_mutexes \
+            --disable-sia \
+            --disable-auth-sasldb \
+            --disable-httpform \
+            --enable-plain \
+            --enable-anon \
+            --enable-login \
+            --disable-ntlm \
+            --disable-passdss \
+            --disable-sql \
+            --disable-ldapdb \
+            --disable-macos-framework
+
+make
+make install 
+ldconfig -v |grep -i sasl 
--- a/ci_scripts/docker-coredeps/deps/build_zlib.sh
+++ b/ci_scripts/docker-coredeps/deps/build_zlib.sh
@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e -x
+
+ZLIB_VERSION=1.2.11
+ZLIB_SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+
+curl -O https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz
+echo "${ZLIB_SHA256}  zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c -
+tar xzf zlib-${ZLIB_VERSION}.tar.gz
+cd zlib-${ZLIB_VERSION} 
+./configure
+make
+make install
+ldconfig -v 
--- a/ci_scripts/docker-coredeps/deps/run_all.sh
+++ b/ci_scripts/docker-coredeps/deps/run_all.sh
@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+#
+set -e -x
+
+# perform clean build of core and install 
+export NINJA_BUILD_DIR=.docker-corebuild
+export TOXWORKDIR=.docker-tox
+[ -d "$NINJA_BUILD_DIR" ] && rm -rf "$NINJA_BUILD_DIR"
+
+meson -Drpgp=true "$NINJA_BUILD_DIR" .
+    
+pushd $NINJA_BUILD_DIR 
+ninja
+ninja install
+ldconfig -v
+popd
+
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+#
+# run ninja and python tests
+#
+
+if [ -n "$TESTS" ]; then 
+    echo ----------------
+    echo run ninja tests
+    echo ----------------
+
+    # ninja test
+
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi
--- a/ci_scripts/docker-doxygen/Dockerfile
+++ b/ci_scripts/docker-doxygen/Dockerfile
@ -0,0 +1,5 @@
+FROM debian:stable
+
+# this is tagged as deltachat/doxygen 
+
+RUN apt-get update && apt-get install -y doxygen 
--- a/ci_scripts/run_all.sh
+++ b/ci_scripts/run_all.sh
@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+# typically run in a docker container that contains all library deps
+# but should also work outside if you have the dependencies installed  
+# on your system. 
+
+set -e -x
+
+# perform clean build of core and install 
+export NINJA_BUILD_DIR=.docker-corebuild
+export TOXWORKDIR=.docker-tox
+[ -d "$NINJA_BUILD_DIR" ] && rm -rf "$NINJA_BUILD_DIR"
+
+meson -Drpgp=true "$NINJA_BUILD_DIR" .
+    
+pushd $NINJA_BUILD_DIR 
+ninja
+ninja install
+ldconfig -v
+popd
+
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then 
+
+    pushd $NINJA_BUILD_DIR
+    # ninja test -> XXX fails because of encoding problem 
+    popd
+    
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi
--- a/python/CHANGELOG
+++ b/python/CHANGELOG
@ -1,3 +1,9 @@
+0.9.1-dev
+---------
+
+- use docker image for building wheels
+- fix code documentation links 
+
 0.9.0
 -----

--- a/python/doc/api.rst
+++ b/python/doc/api.rst
@ -10,9 +10,9 @@ high level API reference
  other classes)
 - :class:`deltachat.chatting.Contact`
 - :class:`deltachat.chatting.Chat`
- :class:`deltachat.chatting.Message`
- :class:`deltachat.chatting.MessageType`
- :class:`deltachat.chatting.MessageState`
+- :class:`deltachat.message.Message`
+- :class:`deltachat.message.MessageType`
+- :class:`deltachat.message.MessageState`

 Account
 -------
@ -36,19 +36,19 @@ Chat
 Message
 -------

-.. autoclass:: deltachat.chatting.Message
+.. autoclass:: deltachat.message.Message
    :members:

 MessageType
 ------------

-.. autoclass:: deltachat.chatting.MessageType
+.. autoclass:: deltachat.message.MessageType
    :members:

 MessageState
 ------------

-.. autoclass:: deltachat.chatting.MessageState
+.. autoclass:: deltachat.message.MessageState
    :members:


--- a/python/src/deltachat/init.py
+++ b/python/src/deltachat/init.py
@ -2,7 +2,7 @@ from deltachat import capi, const
 from deltachat.capi import ffi
 from deltachat.account import Account  # noqa

-__version__ = "0.9.0"
+__version__ = "0.9.1dev1"


 _DC_CALLBACK_MAP = {}
--- a/python/src/deltachat/account.py
+++ b/python/src/deltachat/account.py
@ -141,7 +141,7 @@ class Account(object):

        :param view_type: a string specifying "text", "video",
                          "image", "audio" or "file".
-        :returns: :class:`deltachat.chatting.Message` instance.
+        :returns: :class:`deltachat.message.Message` instance.
        """
        return Message.new(self._dc_context, view_type)

@ -167,7 +167,7 @@ class Account(object):
                      whose name or e-mail matches query.
        :param only_verified: if true only return verified contacts.
        :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.chatting.Message` objects.
+        :returns: list of :class:`deltachat.message.Message` objects.
        """
        flags = 0
        query = as_dc_charpointer(query)
@ -256,7 +256,7 @@ class Account(object):
    def forward_messages(self, messages, chat):
        """ Forward list of messages to a chat.

-        :param messages: list of :class:`deltachat.chatting.Message` object.
+        :param messages: list of :class:`deltachat.message.Message` object.
        :param chat: :class:`deltachat.chatting.Chat` object.
        :returns: None
        """
@ -266,7 +266,7 @@ class Account(object):
    def delete_messages(self, messages):
        """ delete messages (local and remote).

-        :param messages: list of :class:`deltachat.chatting.Message` object.
+        :param messages: list of :class:`deltachat.message.Message` object.
        :returns: None
        """
        msg_ids = [msg.id for msg in messages]
--- a/python/src/deltachat/chatting.py
+++ b/python/src/deltachat/chatting.py
@ -113,7 +113,7 @@ class Chat(object):

        :param msg: unicode text
        :raises: ValueError if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.chatting.Message` instance
+        :returns: the resulting :class:`deltachat.message.Message` instance
        """
        msg = as_dc_charpointer(text)
        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
@ -127,7 +127,7 @@ class Chat(object):
        :param path: path to the file.
        :param mime_type: the mime-type of this file, defaults to application/octet-stream.
        :raises: ValueError if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.chatting.Message` instance
+        :returns: the resulting :class:`deltachat.message.Message` instance
        """
        path = as_dc_charpointer(path)
        mtype = as_dc_charpointer(mime_type)
@ -143,7 +143,7 @@ class Chat(object):

        :param path: path to an image file.
        :raises: ValueError if message can not be send/chat does not exist.
-        :returns: the resulting :class:`deltachat.chatting.Message` instance
+        :returns: the resulting :class:`deltachat.message.Message` instance
        """
        if not os.path.exists(path):
            raise ValueError("path does not exist: {!r}".format(path))
@ -179,7 +179,7 @@ class Chat(object):
        :param message: a :class:`Message` instance previously returned by
                        :meth:`prepare_file`.
        :raises: ValueError if message can not be sent.
-        :returns: a :class:`deltachat.chatting.Message` instance with updated state
+        :returns: a :class:`deltachat.message.Message` instance with updated state
        """
        msg_id = lib.dc_send_msg(self._dc_context, 0, message._dc_msg)
        if msg_id == 0:
@ -189,7 +189,7 @@ class Chat(object):
    def get_messages(self):
        """ return list of messages in this chat.

-        :returns: list of :class:`deltachat.chatting.Message` objects for this chat.
+        :returns: list of :class:`deltachat.message.Message` objects for this chat.
        """
        dc_array = ffi.gc(
            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
--- a/python/src/deltachat/message.py
+++ b/python/src/deltachat/message.py
@ -51,7 +51,7 @@ class Message(object):
    def get_state(self):
        """ get the message in/out state.

-        :returns: :class:`deltachat.chatting.MessageState`
+        :returns: :class:`deltachat.message.MessageState`
        """
        return MessageState(self)

@ -89,7 +89,7 @@ class Message(object):
    def view_type(self):
        """the view type of this message.

-        :returns: a :class:`deltachat.chatting.MessageType` instance.
+        :returns: a :class:`deltachat.message.MessageType` instance.
        """
        return MessageType(lib.dc_msg_get_viewtype(self._dc_msg))

--- a/python/tests/auditwheels.py
+++ b/python/tests/auditwheels.py
@ -0,0 +1,13 @@
+
+import os
+import sys
+import subprocess
+
+
+if __name__ == "__main__":
+    assert len(sys.argv) == 2
+    workspacedir = sys.argv[1]
+    for relpath in os.listdir(workspacedir):
+        if relpath.startswith("deltachat"):
+            p = os.path.join(workspacedir, relpath)
+            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])
--- a/python/tox.ini
+++ b/python/tox.ini
@ -4,15 +4,25 @@ envlist =
    py27
    py35
    lint
+    auditwheels

 [testenv]
-commands = pytest -rsXx {posargs:tests}
+commands = 
+    pytest -rsXx {posargs:tests}
+    pip wheel . -w {toxworkdir}/wheelhouse
+ 
 passenv = TRAVIS
 deps = 
    pytest
    pytest-faulthandler
    pdbpp

+[testenv:auditwheels]
+skipsdist = True
+commands = 
+    python tests/auditwheels.py {toxworkdir}/wheelhouse
+
+
 [testenv:lint]
 skipsdist = True
 usedevelop = True
@ -28,14 +38,13 @@ commands =
    rst-lint --encoding 'utf-8' README.rst

 [testenv:doc]
-usedevelop = True
 deps =
-    sphinx
+    sphinx==1.8.5
    breathe

 changedir = doc
 commands =
-    sphinx-build -b html . _build/html
+    sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html


 [pytest]
--- a/python/wheelbuilder/Dockerfile
+++ b/python/wheelbuilder/Dockerfile
@ -1,123 +0,0 @@
-FROM quay.io/pypa/manylinux1_x86_64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install meson and ninja
-ENV NINJA_VERSION v1.8.2
-ENV NINJA_SHA256 d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07
-RUN curl -L -o ninja-linux-$NINJA_VERSION.zip https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip
-RUN echo "${NINJA_SHA256}  ninja-linux-${NINJA_VERSION}.zip" | sha256sum -c -
-RUN unzip ninja-linux-${NINJA_VERSION}.zip
-RUN mv ninja /usr/bin/ninja
-RUN /opt/python/cp37-cp37m/bin/pip install meson
-RUN cd /usr/bin && ln -s /opt/_internal/cpython-3.7.*/bin/meson
-
-# Install a recent zlib, needed by libetpan
-ENV ZLIB_VERSION 1.2.11
-ENV ZLIB_SHA256 c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-RUN curl -O https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz
-RUN echo "${ZLIB_SHA256}  zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c -
-RUN tar xzf zlib-${ZLIB_VERSION}.tar.gz
-RUN cd zlib-${ZLIB_VERSION} && ./configure
-RUN cd zlib-${ZLIB_VERSION} && make
-RUN cd zlib-${ZLIB_VERSION} && make install
-RUN ldconfig -v
-
-# Install a recent Perl, needed to install OpenSSL
-ENV PERL_VERSION 5.28.0
-ENV PERL_SHA256 7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
-RUN curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-RUN echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-RUN tar xzf perl-${PERL_VERSION}.tar.gz
-RUN cd perl-${PERL_VERSION} && ./Configure -de
-RUN cd perl-${PERL_VERSION} && make
-RUN cd perl-${PERL_VERSION} && make install
-
-# Install OpenSSL
-ENV OPENSSL_VERSION 1.1.1a
-ENV OPENSSL_SHA256 fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-RUN curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-RUN echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-RUN tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-RUN cd openssl-${OPENSSL_VERSION} && \
-    ./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-RUN cd openssl-${OPENSSL_VERSION} && \
-    sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-    sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-    sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-RUN cd openssl-${OPENSSL_VERSION} && make depend
-RUN cd openssl-${OPENSSL_VERSION} && make
-RUN cd openssl-${OPENSSL_VERSION} && make install_sw install_ssldirs
-RUN ldconfig -v
-
-# Install cyrus-sasl
-ENV SASL_VERSION 2.1.27
-ENV SASL_SHA256 26866b1549b00ffd020f188a43c258017fa1c382b3ddadd8201536f72efb05d5
-RUN curl -O https://www.cyrusimap.org/releases/cyrus-sasl-${SASL_VERSION}.tar.gz
-RUN echo "${SASL_SHA256}  cyrus-sasl-${SASL_VERSION}.tar.gz" | sha256sum -c -
-RUN tar zxf cyrus-sasl-${SASL_VERSION}.tar.gz
-RUN cd cyrus-sasl-${SASL_VERSION} && \
-    ./configure --disable-silent-rules \
-                --disable-cmulocal \
-                --disable-sample \
-                --disable-obsolete_cram_attr \
-                --disable-obsolete_digest_attr \
-                --disable-staticdlopen \
-                --disable-java \
-                --disable-alwaystrue \
-                --enable-checkapop \
-                --enable-cram \
-                --enable-digest \
-                --enable-scram \
-                --disable-otp \
-                --disable-srp \
-                --disable-srp-setpass \
-                --disable-krb4 \
-                --disable-gssapi \
-                --disable-gss_mutexes \
-                --disable-sia \
-                --disable-auth-sasldb \
-                --disable-httpform \
-                --enable-plain \
-                --enable-anon \
-                --enable-login \
-                --disable-ntlm \
-                --disable-passdss \
-                --disable-sql \
-                --disable-ldapdb \
-                --disable-macos-framework
-RUN cd cyrus-sasl-${SASL_VERSION} && make
-RUN cd cyrus-sasl-${SASL_VERSION} && make install
-RUN ldconfig -v
-
-# Install libetpan
-ENV ETPAN_VERSION 1.9.1
-ENV ETPAN_SHA256 f5e354ccf1014c6ee313ade1009b8a82f28043d2504655e388bb4c1328700fcd
-RUN curl -L -o libetpan-${ETPAN_VERSION}.tar.gz \
-    https://github.com/dinhviethoa/libetpan/archive/${ETPAN_VERSION}.tar.gz
-RUN echo "${ETPAN_SHA256}  libetpan-${ETPAN_VERSION}.tar.gz" | sha256sum -c -
-RUN tar xzf libetpan-${ETPAN_VERSION}.tar.gz
-RUN cd libetpan-${ETPAN_VERSION} && \
-    ./autogen.sh && \
-    ./configure --enable-ipv6 \
-                --enable-iconv --disable-db \
-                --with-openssl --with-sasl --with-zlib \
-                --without-curl --without-expat
-RUN cd libetpan-${ETPAN_VERSION} && make
-RUN cd libetpan-${ETPAN_VERSION} && make install
-RUN ldconfig -v
-
-# Install Rust nightly 
-RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y
-ENV PATH=/root/.cargo/bin:$PATH
-RUN rustc --version
-
-# Install RPGP from github -- this currently downloads 500MB
-# see reported issue: https://github.com/dignifiedquire/rpgp/issues/30 
-RUN true \
-    && git clone https://github.com/dignifiedquire/rpgp.git \
-    && cd rpgp/pgp-ffi \
-    && make install
--- a/python/wheelbuilder/build-wheels.sh
+++ b/python/wheelbuilder/build-wheels.sh
@ -1,27 +0,0 @@
-#!/bin/bash
-set -e -x
-
-## Build the library
-meson -Drpgp=true /builddir /io
-pushd /builddir
-ninja
-ninja install
-ldconfig -v
-popd
-
-## Compile wheels
-for PYBIN in /opt/python/*/bin; do
-    "${PYBIN}/pip" install cffi requests attrs six pytest
-    "${PYBIN}/pip" wheel /io/python -w wheelhouse/
-done
-
-## Bundle external shared libraries into the wheels
-for whl in wheelhouse/deltachat*.whl; do
-    auditwheel repair "$whl" -w /io/python/wheelhouse/
-done
-
-## Install packages (and test)
-for PYBIN in /opt/python/*/bin/; do
-    "${PYBIN}/pip" install deltachat --no-index -f /io/python/wheelhouse
-    # (cd "$HOME"; "${PYBIN}/pytest" /io/python/tests)
-done