diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 120000 index 00000000..2c42d1a2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1 @@ +../ci_scripts/config.yml \ No newline at end of file diff --git a/.gitignore b/.gitignore index 680c1469..c96dd44b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ # ignore backup files *~ +# ignore tox and docker stuff +.docker-* +wheelhouse +workspace + # ignore build temporary files *.o .deps diff --git a/.scripts/buildwheel.sh b/.scripts/buildwheel.sh deleted file mode 100644 index 91dd22e7..00000000 --- a/.scripts/buildwheel.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e -x - -docker pull deltachat/wheel -rm -rf python/wheelhouse/* -cd $TRAVIS_BUILD_DIR -docker run --rm -it -v $(pwd):/io deltachat/wheel /io/python/wheelbuilder/build-wheels.sh - -# create an index at the community "devpi" python packaging site -# and push both binary wheel packages and the source package to -# the https://m.devpi.net/dc/BRANCHNAME index -devpi use https://m.devpi.net -devpi login dc --password $DEVPI_LOGIN - -devpi use dc/$BRANCH || { - devpi index -c $BRANCH - devpi use dc/$BRANCH -} -devpi index $BRANCH bases=/root/pypi - -devpi upload --from-dir python/wheelhouse - -cd python -devpi upload diff --git a/.scripts/deploy.sh b/.scripts/deploy.sh deleted file mode 100755 index d80d2bb1..00000000 --- a/.scripts/deploy.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -e -set -u -set -x -set -v - - -#Only attempt to deploy if we know the ssh key secrets, username and server -if test -z ${encrypted_49475b8073e9_key:+decryp_key} ; then exit 0; fi -if test -z ${encrypted_49475b8073e9_iv:+decrypt_iv} ; then exit 0; fi -if test -z ${DEPLOY_USER:+username} ; then exit 0; fi -if test -z ${DEPLOY_SERVER:+server} ; then exit 0; fi - -# Prepare the ssh homedir. -# -# Decrypting the ssh private key for deploying to the server. -# See https://docs.travis-ci.com/user/encrypting-files/ for details. -mkdir -p -m 0700 ~/.ssh -openssl aes-256-cbc \ - -K $encrypted_49475b8073e9_key \ - -iv $encrypted_49475b8073e9_iv \ - -in $TRAVIS_BUILD_DIR/.credentials/delta.id_rsa.enc \ - -out ~/.ssh/id_rsa -d -chmod 600 ~/.ssh/id_rsa -printf "Host *\n" >> ~/.ssh/config -printf " %sAuthentication no\n" ChallengeResponse Password KbdInteractive >> ~/.ssh/config - - -# Perform the actual deploy to py.delta.chat -rsync -avz \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - $TRAVIS_BUILD_DIR/python/doc/_build/html/ \ - ${DEPLOY_USER}@${DEPLOY_SERVER}:build/${TRAVIS_BRANCH/\//_} - -# Perform the actual deploy to c.delta.chat -rsync -avz \ - -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ - $TRAVIS_BUILD_DIR/docs/html/ \ - ${DEPLOY_USER}@${DEPLOY_SERVER}:build-c/${TRAVIS_BRANCH/\//_} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d45e4a24..00000000 --- a/.travis.yml +++ /dev/null @@ -1,70 +0,0 @@ -sudo: required -language: c -env: - global: - - DEPLOY_USER=delta DEPLOY_SERVER="py.delta.chat" - - secure: kxF/4pnq6jnhc/9BCtO8kQnoFg6w3/SMfy4zE54pqCzJEHKmOTVV0B+XX7epwYqOMI3aDDprVp+3+qi9+soz8ISN2Sedu1ZRje2ILv/OhHJUVkMLgAmDzzPBhlvlVXA18baIohxjJLGFHw2ZLz9aKHZENcM33ujs/4bSdODuWw6Re8C29aolh6+gvDlb/TmyB27GZoFa3tyc0EU0IhEYX4sPMkkFJVefvTPeKlQzKwJ0InfGMoWhlpr8CqFLsPHbcbDGrkoWXmmMOQny49itsr0AQoe9jpRnHAejrunOKlEnB5q+ok4p/1AQb5K7rZ0m93jbgyjT8Gz2GOeM49xd5M75ZzaLJXs6EkaWg7Y8kP7d7QpkXmKeQK+VjP1n7k2u+hGl9v89mk5aDvhceBBr4v0CjfLZKL3INd2/Z8CBggMyjpGhri5BCNOHxlFycqnsGv0ssz+BSElGQ8KeKtMVfqxkdikPTSpRCX+FT7DfZbM8KiDk0ezI3dEDRBMSZtQ4fngxdasC5FfrH8Y6g6LxAkByfwpGij6Rio0A7ICjlg8b0sJ7iEiFHy5lfkMSm+yoSEzyjFQb/D9JxnsnCTfqL+qr8j3IGGswEnDP9iRkw156gHneFsOMYW6wOtci7ZzYxdxTCGRt9y2HQiQQn8VgI9fLI77/ijDQmt13s1StikE= - matrix: - - MESONARGS="" DOCS=t TESTS=t WHEEL=t - - MESONARGS="-Dmonolith=true" TESTS=t - - MESONARGS="--default-library=static" - - MESONARGS="--wrap-mode=forcefallback --default-library=static" -addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - python3-pip - - python3.5 - - g++-7 - - libssl-dev - - libsqlite3-dev - - libbz2-dev - - zlib1g-dev - - python3.5-dev - - python-software-properties - - doxygen - -services: - - docker - -install: - - export CC=gcc-7 - - export CXX=g++-7 - - sudo ln -sf /usr/bin/python3.5 /usr/bin/python3 - - sudo pip3 install meson - - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip - - unzip ninja-linux.zip - - sudo cp ninja /usr/bin - - wget http://http.debian.net/debian/pool/main/c/cyrus-sasl2/cyrus-sasl2_2.1.27~101-g0780600+dfsg.orig.tar.xz - - tar xfv cyrus-sasl2_2.1.27~101-g0780600+dfsg.orig.tar.xz - - pushd cyrus-sasl2-2.1.27~101-g0780600+dfsg.orig && ./autogen.sh && make && sudo - make install && popd - -script: - - export BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} - - echo "BRANCH=$BRANCH" - - doxygen --version - - mkdir -p builddir && pushd builddir - - meson $MESONARGS && ninja -v && sudo ninja install - - export LD_LIBRARY_PATH=/usr/local/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH - - sudo ldconfig -v - - popd && pushd docs - - if [ -n "$DOCS" ]; then doxygen; fi - - popd && pushd python - - virtualenv -p /usr/bin/python3.5 venv - - source venv/bin/activate - - pip install tox devpi-client - - if [ -e /usr/local/lib/x86_64-linux-gnu/libdeltachat.so ]; then ldd /usr/local/lib/x86_64-linux-gnu/libdeltachat.so; - fi - - if [ -n "$TESTS" ]; then tox; fi - - if [ -n "$DOCS" ]; then tox -e doc; fi - - if [ -n "$WHEEL" ]; then bash $TRAVIS_BUILD_DIR/.scripts/buildwheel.sh ; fi - - popd -deploy: - provider: script - skip_cleanup: true - script: bash $TRAVIS_BUILD_DIR/.scripts/deploy.sh - on: - all_branches: true - condition: $DOCS diff --git a/README.md b/README.md index 4a99d436..d4d74e7e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Delta Chat Core Library -[![Build Status](https://travis-ci.org/deltachat/deltachat-core.svg?branch=master)](https://travis-ci.org/deltachat/deltachat-core) +[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core.svg?style=svg)](https://circleci.com/gh/deltachat/deltachat-core) The _Delta Chat Core Library_ is written in cross-platform **C**, documented at . @@ -9,6 +9,9 @@ The ``deltachat`` Python bindings can be found in the [python subdirectory](https://github.com/deltachat/deltachat-core/tree/master/python) and are documented at . +In the [ci_scripts directory](https://github.com/deltachat/deltachat-core/tree/master/ci_scripts/README.md) +you'll find docker- and library-building scripts. + ## binary/distribution packages (work-in-progress) There are work-in-progress efforts for creating (binary) packages which @@ -16,6 +19,8 @@ do not require that you build the library manually: - [libdeltachat-core-git archlinux package](https://aur.archlinux.org/packages/libdeltachat-core-git/>) +- [python-wheel packaging](https://m.devpi.net/dc/master) + - [Debian packaging](https://github.com/deltachat/deltachat-core/issues/299) - [Windows building](https://github.com/deltachat/deltachat-core/issues/306) diff --git a/ci_scripts/README.md b/ci_scripts/README.md new file mode 100644 index 00000000..6a239754 --- /dev/null +++ b/ci_scripts/README.md @@ -0,0 +1,52 @@ + +# Continuous Integration Scripts for Delta Chat + +Continuous Integration is run through CircleCI +but is largely independent of it. + + +## Generating docker containers for performing build step work + +All tests, docs and wheel building is run in docker containers: + +- **coredeps/Dockerfile** specifies an image that contains all + of Delta Chat's core dependencies as linkable libraries. + It also serves to run python tests and build wheels + (binary packages for Python). + +- **doxygen/Dockerfile** specifies an image that contains + the doxygen tool which is used to generate C-docs. + +To run tests locally you can pull existing images from "docker.io", +the hub for sharing Docker images:: + + docker pull deltachat/coredeps + docker pull deltachat/doxygen + +or you can build the docker images yourself locally +to avoid the relatively large download:: + + cd ci_scripts # where all CI things are + docker build -t deltachat/coredeps docker-coredeps + docker build -t deltachat/doxygen docker-doxygen + +## ci_run.sh (main entrypoint called by circle-ci) + +Once you have the docker images available +you can run python testing, documentation generation +and building binary wheels:: + + sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh + +## ci_upload.sh (uploading artifacts on success) + +- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/` + +- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/` + +- python wheels to `https://m.devpi.net/dc/` + so that you install fully self-contained wheels like this: + `pip install -U -i https://m.devpi.net/dc/ deltachat` + + + diff --git a/ci_scripts/ci_run.sh b/ci_scripts/ci_run.sh new file mode 100755 index 00000000..0d6c35e2 --- /dev/null +++ b/ci_scripts/ci_run.sh @@ -0,0 +1,20 @@ +# perform CI jobs on PRs and after merges to master. +# triggered from .circleci/config.yml + +set -e -x + +export BRANCH=${CIRCLE_BRANCH:-test7} + +# run doxygen on c-source (needed by later doc-generation steps). +# XXX modifies the host filesystem docs/xml and docs/html directories +# XXX which you can then only remove with sudo as they belong to root + +if [ -n "$DOCS" ] ; then + docker run --rm -it -v $PWD:/mnt -w /mnt/docs deltachat/doxygen doxygen +fi + +# run everything else inside docker (TESTS, DOCS, WHEELS) +docker run -e BRANCH -e MESONARGS -e TESTS -e DOCS \ + --rm -it -v $(pwd):/mnt -w /mnt \ + deltachat/coredeps ci_scripts/run_all.sh + diff --git a/ci_scripts/ci_upload.sh b/ci_scripts/ci_upload.sh new file mode 100755 index 00000000..8bdac4a4 --- /dev/null +++ b/ci_scripts/ci_upload.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +if [ -z "$DEVPI_LOGIN" ] ; then + echo "required: password for 'dc' user on https://m.devpi/net/dc index" + exit 1 +fi + +set -xe + +DOXYDOCDIR=${1:?directory where doxygen docs to be found} +PYDOCDIR=${2:?directory with python docs} +WHEELHOUSEDIR=${3:?directory with pre-built wheels} + +export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes} + + +# python docs to py.delta.chat +rsync -avz \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + "$PYDOCDIR/html/" \ + delta@py.delta.chat:build/${BRANCH} + +# C docs to c.delta.chat +rsync -avz \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + "$DOXYDOCDIR/html/" \ + delta@py.delta.chat:build-c/${BRANCH} + +echo ----------------------- +echo upload wheels +echo ----------------------- + +# Bundle external shared libraries into the wheels +pushd $WHEELHOUSEDIR + +pip install devpi-client +devpi use https://m.devpi.net +devpi login dc --password $DEVPI_LOGIN + +devpi use dc/$BRANCH || { + devpi index -c $BRANCH + devpi use dc/$BRANCH +} +devpi index $BRANCH bases=/root/pypi +devpi upload deltachat*.whl + +popd diff --git a/ci_scripts/config.yml b/ci_scripts/config.yml new file mode 100644 index 00000000..da06d158 --- /dev/null +++ b/ci_scripts/config.yml @@ -0,0 +1,90 @@ + +version: 2 +jobs: + build_test_docs_wheel: + machine: True + steps: + - checkout + - run: docker pull deltachat/doxygen + - run: docker pull deltachat/coredeps + - run: + command: ci_scripts/ci_run.sh + environment: + MESONARGS: + DOCS: 1 + TESTS: 1 + + - run: + name: copying C docs, python docs and wheels to workspace + command: | + mkdir -p workspace/python + cp -av docs workspace/c-docs + cp -av python/.docker-tox/wheelhouse workspace/ + cp -av python/doc/_build/ workspace/py-docs + + - persist_to_workspace: + root: workspace + paths: + - c-docs + - py-docs + - wheelhouse + + upload_docs_wheels: + machine: True + steps: + - run: + command: | + echo 'export MESONARGS=""' >> $BASH_ENV + - checkout + - attach_workspace: + at: workspace + - run: ls -laR workspace + - run: ci_scripts/ci_upload.sh workspace/c-docs workspace/py-docs workspace/wheelhouse + + build_monolith_tests: + machine: True + steps: + - checkout + - run: docker pull deltachat/coredeps + - run: + command: | + echo 'export MESONARGS="-Dmonolith=true"' >> $BASH_ENV + echo 'export TESTS=1' >> $BASH_ENV + - run: ci_scripts/ci_run.sh + + build_static: + machine: True + steps: + - checkout + - run: docker pull deltachat/coredeps + - run: + command: | + echo 'export MESONARGS="--default-library=static"' >> $BASH_ENV + - run: ci_scripts/ci_run.sh + + build_forcefallback: + machine: True + steps: + - checkout + - run: docker pull deltachat/coredeps + - run: + command: | + echo 'export MESONARGS="--wrap-mode=forcefallback --default-library=static"' >> $BASH_ENV + - run: ci_scripts/ci_run.sh + +workflows: + version: 2 + build_all: + jobs: + - build_test_docs_wheel + - build_monolith_tests + - build_static + - build_forcefallback + - upload_docs_wheels: + requires: + - build_test_docs_wheel + - build_monolith_tests + - build_static + - build_forcefallback + + diff --git a/ci_scripts/docker-coredeps/Dockerfile b/ci_scripts/docker-coredeps/Dockerfile new file mode 100644 index 00000000..a7973b60 --- /dev/null +++ b/ci_scripts/docker-coredeps/Dockerfile @@ -0,0 +1,44 @@ +FROM quay.io/pypa/manylinux1_x86_64 + +# Configure ld.so/ldconfig and pkg-config +RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \ + echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf +ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig + +ENV PIP_DISABLE_PIP_VERSION_CHECK 1 + +# Install ninja +ADD deps/build_ninja.sh /scripts/build_ninja.sh +RUN mkdir tmp1 && cd tmp1 && bash /scripts/build_ninja.sh && cd .. && rm -r tmp1 + +# Install a recent zlib, needed by libetpan +ADD deps/build_zlib.sh /builder/build_zlib.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_zlib.sh && cd .. && rm -r tmp1 + +# Install a recent Perl, needed to install OpenSSL +ADD deps/build_perl.sh /builder/build_perl.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1 + +# Install OpenSSL +ADD deps/build_openssl.sh /builder/build_openssl.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1 + +# Install cyrus-sasl +ADD deps/build_sasl.sh /builder/build_sasl.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_sasl.sh && cd .. && rm -r tmp1 + +# Install libetpan +ADD deps/build_libetpan.sh /builder/build_libetpan.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_libetpan.sh && cd .. && rm -r tmp1 + +# Install python tools (meson, tox, ...) +ADD deps/build_python.sh /builder/build_python.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1 + +# Install Rust nightly +ADD deps/build_rust.sh /builder/build_rust.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1 + +# Install RPGP from current github (beware, will be static in the docker image) +ADD deps/build_rpgp.sh /builder/build_rpgp.sh +RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rpgp.sh && cd .. && rm -r tmp1 diff --git a/ci_scripts/docker-coredeps/deps/build_libetpan.sh b/ci_scripts/docker-coredeps/deps/build_libetpan.sh new file mode 100755 index 00000000..e0db72e0 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_libetpan.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e -x + +ETPAN_VERSION=1.9.1 +ETPAN_SHA256=f5e354ccf1014c6ee313ade1009b8a82f28043d2504655e388bb4c1328700fcd +curl -L -o libetpan-${ETPAN_VERSION}.tar.gz \ + https://github.com/dinhviethoa/libetpan/archive/${ETPAN_VERSION}.tar.gz +echo "${ETPAN_SHA256} libetpan-${ETPAN_VERSION}.tar.gz" | sha256sum -c - +tar xzf libetpan-${ETPAN_VERSION}.tar.gz +cd libetpan-${ETPAN_VERSION} +./autogen.sh && \ +./configure --enable-ipv6 \ + --enable-iconv --disable-db \ + --with-openssl --with-sasl --with-zlib \ + --without-curl --without-expat +make +make install +ldconfig -v | grep -i etpan diff --git a/ci_scripts/docker-coredeps/deps/build_ninja.sh b/ci_scripts/docker-coredeps/deps/build_ninja.sh new file mode 100755 index 00000000..ac376398 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_ninja.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e -x + +NINJA_VERSION=v1.8.2 +NINJA_SHA256=d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07 + +curl -L -o ninja-linux-$NINJA_VERSION.zip \ + https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip + +echo "${NINJA_SHA256} ninja-linux-${NINJA_VERSION}.zip" | sha256sum -c - +unzip ninja-linux-${NINJA_VERSION}.zip +mv ninja /usr/bin/ninja + diff --git a/ci_scripts/docker-coredeps/deps/build_openssl.sh b/ci_scripts/docker-coredeps/deps/build_openssl.sh new file mode 100755 index 00000000..6f609186 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_openssl.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e -x + +OPENSSL_VERSION=1.1.1a +OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41 + +curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz +echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c - +tar xzf openssl-${OPENSSL_VERSION}.tar.gz +cd openssl-${OPENSSL_VERSION} +./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local + +sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \ +sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \ +sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile + +make depend +make +make install_sw install_ssldirs +ldconfig -v | grep ssl diff --git a/ci_scripts/docker-coredeps/deps/build_perl.sh b/ci_scripts/docker-coredeps/deps/build_perl.sh new file mode 100755 index 00000000..4acb51c3 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_perl.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +PERL_VERSION=5.28.0 +PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8 +curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz +echo "${PERL_SHA256} perl-${PERL_VERSION}.tar.gz" | sha256sum -c - +tar xzf perl-${PERL_VERSION}.tar.gz +cd perl-${PERL_VERSION} + +./Configure -de +make +make install diff --git a/ci_scripts/docker-coredeps/deps/build_python.sh b/ci_scripts/docker-coredeps/deps/build_python.sh new file mode 100755 index 00000000..0aecc223 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_python.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -x -e + +# we use the python3.5 environment as the base environment +/opt/python/cp35-cp35m/bin/pip install meson tox devpi-client auditwheel + +pushd /usr/bin + +ln -s /opt/_internal/cpython-3.5.*/bin/meson +ln -s /opt/_internal/cpython-3.5.*/bin/tox +ln -s /opt/_internal/cpython-3.5.*/bin/devpi +ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel + +popd diff --git a/ci_scripts/docker-coredeps/deps/build_rpgp.sh b/ci_scripts/docker-coredeps/deps/build_rpgp.sh new file mode 100755 index 00000000..2813ca22 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_rpgp.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e -x + +# Install RPGP from github +# XXX built a particular version like the other dep scripts do + +export PATH=$PATH:$HOME/.cargo/bin +git clone https://github.com/rpgp/rpgp.git +cd rpgp/pgp-ffi +make install + +# after we installed the RPGP lib we don't need Rust anymore +# rm -rf /root/.cargo /root.rustup diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh new file mode 100755 index 00000000..f1356f7e --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_rust.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e -x + +# Install Rust +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y +export PATH=/root/.cargo/bin:$PATH +rustc --version + +# remove some 300-400 MB that we don't need for automated builds +rm -rf /root/.rustup/toolchains/nightly-2019-03-23-x86_64-unknown-linux-gnu/share/ diff --git a/ci_scripts/docker-coredeps/deps/build_sasl.sh b/ci_scripts/docker-coredeps/deps/build_sasl.sh new file mode 100755 index 00000000..2c55182e --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_sasl.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e -x + +SASL_VERSION=2.1.27 +SASL_SHA256=26866b1549b00ffd020f188a43c258017fa1c382b3ddadd8201536f72efb05d5 + +curl -O https://www.cyrusimap.org/releases/cyrus-sasl-${SASL_VERSION}.tar.gz +echo "${SASL_SHA256} cyrus-sasl-${SASL_VERSION}.tar.gz" | sha256sum -c - +tar zxf cyrus-sasl-${SASL_VERSION}.tar.gz + +cd cyrus-sasl-${SASL_VERSION} + +./configure --disable-silent-rules \ + --disable-cmulocal \ + --disable-sample \ + --disable-obsolete_cram_attr \ + --disable-obsolete_digest_attr \ + --disable-staticdlopen \ + --disable-java \ + --disable-alwaystrue \ + --enable-checkapop \ + --enable-cram \ + --enable-digest \ + --enable-scram \ + --disable-otp \ + --disable-srp \ + --disable-srp-setpass \ + --disable-krb4 \ + --disable-gssapi \ + --disable-gss_mutexes \ + --disable-sia \ + --disable-auth-sasldb \ + --disable-httpform \ + --enable-plain \ + --enable-anon \ + --enable-login \ + --disable-ntlm \ + --disable-passdss \ + --disable-sql \ + --disable-ldapdb \ + --disable-macos-framework + +make +make install +ldconfig -v |grep -i sasl diff --git a/ci_scripts/docker-coredeps/deps/build_zlib.sh b/ci_scripts/docker-coredeps/deps/build_zlib.sh new file mode 100755 index 00000000..234e6530 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/build_zlib.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e -x + +ZLIB_VERSION=1.2.11 +ZLIB_SHA256=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 + +curl -O https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz +echo "${ZLIB_SHA256} zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c - +tar xzf zlib-${ZLIB_VERSION}.tar.gz +cd zlib-${ZLIB_VERSION} +./configure +make +make install +ldconfig -v diff --git a/ci_scripts/docker-coredeps/deps/run_all.sh b/ci_scripts/docker-coredeps/deps/run_all.sh new file mode 100755 index 00000000..41f4fd51 --- /dev/null +++ b/ci_scripts/docker-coredeps/deps/run_all.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Build the Delta Chat C/Rust library +# +set -e -x + +# perform clean build of core and install +export NINJA_BUILD_DIR=.docker-corebuild +export TOXWORKDIR=.docker-tox +[ -d "$NINJA_BUILD_DIR" ] && rm -rf "$NINJA_BUILD_DIR" + +meson -Drpgp=true "$NINJA_BUILD_DIR" . + +pushd $NINJA_BUILD_DIR +ninja +ninja install +ldconfig -v +popd + + +# configure access to a base python and +# to several python interpreters needed by tox below +export PATH=$PATH:/opt/python/cp35-cp35m/bin +export PYTHONDONTWRITEBYTECODE=1 +pushd /bin +ln -s /opt/python/cp27-cp27m/bin/python2.7 +ln -s /opt/python/cp36-cp36m/bin/python3.6 +ln -s /opt/python/cp37-cp37m/bin/python3.7 +popd + +# +# run ninja and python tests +# + +if [ -n "$TESTS" ]; then + echo ---------------- + echo run ninja tests + echo ---------------- + + # ninja test + + echo ---------------- + echo run python tests + echo ---------------- + + pushd python + # first run all tests ... + rm -rf tests/__pycache__ + rm -rf src/deltachat/__pycache__ + export PYTHONDONTWRITEBYTECODE=1 + tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37 + popd +fi + + +if [ -n "$DOCS" ]; then + echo ----------------------- + echo generating python docs + echo ----------------------- + (cd python && tox --workdir "$TOXWORKDIR" -e doc) +fi diff --git a/ci_scripts/docker-doxygen/Dockerfile b/ci_scripts/docker-doxygen/Dockerfile new file mode 100644 index 00000000..c931294f --- /dev/null +++ b/ci_scripts/docker-doxygen/Dockerfile @@ -0,0 +1,5 @@ +FROM debian:stable + +# this is tagged as deltachat/doxygen + +RUN apt-get update && apt-get install -y doxygen diff --git a/ci_scripts/run_all.sh b/ci_scripts/run_all.sh new file mode 100755 index 00000000..72a70996 --- /dev/null +++ b/ci_scripts/run_all.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Build the Delta Chat C/Rust library +# typically run in a docker container that contains all library deps +# but should also work outside if you have the dependencies installed +# on your system. + +set -e -x + +# perform clean build of core and install +export NINJA_BUILD_DIR=.docker-corebuild +export TOXWORKDIR=.docker-tox +[ -d "$NINJA_BUILD_DIR" ] && rm -rf "$NINJA_BUILD_DIR" + +meson -Drpgp=true "$NINJA_BUILD_DIR" . + +pushd $NINJA_BUILD_DIR +ninja +ninja install +ldconfig -v +popd + + +# configure access to a base python and +# to several python interpreters needed by tox below +export PATH=$PATH:/opt/python/cp35-cp35m/bin +export PYTHONDONTWRITEBYTECODE=1 +pushd /bin +ln -s /opt/python/cp27-cp27m/bin/python2.7 +ln -s /opt/python/cp36-cp36m/bin/python3.6 +ln -s /opt/python/cp37-cp37m/bin/python3.7 +popd + +if [ -n "$TESTS" ]; then + + pushd $NINJA_BUILD_DIR + # ninja test -> XXX fails because of encoding problem + popd + + echo ---------------- + echo run python tests + echo ---------------- + + pushd python + # first run all tests ... + rm -rf tests/__pycache__ + rm -rf src/deltachat/__pycache__ + export PYTHONDONTWRITEBYTECODE=1 + tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37 + popd +fi + + +if [ -n "$DOCS" ]; then + echo ----------------------- + echo generating python docs + echo ----------------------- + (cd python && tox --workdir "$TOXWORKDIR" -e doc) +fi diff --git a/meson.build b/meson.build index 44f45317..dfec6336 100644 --- a/meson.build +++ b/meson.build @@ -112,7 +112,7 @@ if get_option('rpgp') # The rpgp pkg-config file is currently incorrect, so just try # linking it instead for now. rpgp = dependency('rpgp') - #rpgp = cc.find_library('rpgp') + # rpgp = cc.find_library('rpgp') add_project_arguments('-DDC_USE_RPGP', language: 'c') netpgp = dependency('', required: false) diff --git a/python/CHANGELOG b/python/CHANGELOG index 3ec3c9c4..87f6449c 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -1,3 +1,9 @@ +0.9.1-dev +--------- + +- use docker image for building wheels +- fix code documentation links + 0.9.0 ----- diff --git a/python/doc/api.rst b/python/doc/api.rst index 7a6c003e..4e8e1e9a 100644 --- a/python/doc/api.rst +++ b/python/doc/api.rst @@ -10,9 +10,9 @@ high level API reference other classes) - :class:`deltachat.chatting.Contact` - :class:`deltachat.chatting.Chat` -- :class:`deltachat.chatting.Message` -- :class:`deltachat.chatting.MessageType` -- :class:`deltachat.chatting.MessageState` +- :class:`deltachat.message.Message` +- :class:`deltachat.message.MessageType` +- :class:`deltachat.message.MessageState` Account ------- @@ -36,19 +36,19 @@ Chat Message ------- -.. autoclass:: deltachat.chatting.Message +.. autoclass:: deltachat.message.Message :members: MessageType ------------ -.. autoclass:: deltachat.chatting.MessageType +.. autoclass:: deltachat.message.MessageType :members: MessageState ------------ -.. autoclass:: deltachat.chatting.MessageState +.. autoclass:: deltachat.message.MessageState :members: diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 6d319872..8c6591d1 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -2,7 +2,7 @@ from deltachat import capi, const from deltachat.capi import ffi from deltachat.account import Account # noqa -__version__ = "0.9.0" +__version__ = "0.9.1dev1" _DC_CALLBACK_MAP = {} diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 5a8ab458..963fc023 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -141,7 +141,7 @@ class Account(object): :param view_type: a string specifying "text", "video", "image", "audio" or "file". - :returns: :class:`deltachat.chatting.Message` instance. + :returns: :class:`deltachat.message.Message` instance. """ return Message.new(self._dc_context, view_type) @@ -167,7 +167,7 @@ class Account(object): whose name or e-mail matches query. :param only_verified: if true only return verified contacts. :param with_self: if true the self-contact is also returned. - :returns: list of :class:`deltachat.chatting.Message` objects. + :returns: list of :class:`deltachat.message.Message` objects. """ flags = 0 query = as_dc_charpointer(query) @@ -256,7 +256,7 @@ class Account(object): def forward_messages(self, messages, chat): """ Forward list of messages to a chat. - :param messages: list of :class:`deltachat.chatting.Message` object. + :param messages: list of :class:`deltachat.message.Message` object. :param chat: :class:`deltachat.chatting.Chat` object. :returns: None """ @@ -266,7 +266,7 @@ class Account(object): def delete_messages(self, messages): """ delete messages (local and remote). - :param messages: list of :class:`deltachat.chatting.Message` object. + :param messages: list of :class:`deltachat.message.Message` object. :returns: None """ msg_ids = [msg.id for msg in messages] diff --git a/python/src/deltachat/chatting.py b/python/src/deltachat/chatting.py index ccca6be5..ab0bdaba 100644 --- a/python/src/deltachat/chatting.py +++ b/python/src/deltachat/chatting.py @@ -113,7 +113,7 @@ class Chat(object): :param msg: unicode text :raises: ValueError if message can not be send/chat does not exist. - :returns: the resulting :class:`deltachat.chatting.Message` instance + :returns: the resulting :class:`deltachat.message.Message` instance """ msg = as_dc_charpointer(text) msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg) @@ -127,7 +127,7 @@ class Chat(object): :param path: path to the file. :param mime_type: the mime-type of this file, defaults to application/octet-stream. :raises: ValueError if message can not be send/chat does not exist. - :returns: the resulting :class:`deltachat.chatting.Message` instance + :returns: the resulting :class:`deltachat.message.Message` instance """ path = as_dc_charpointer(path) mtype = as_dc_charpointer(mime_type) @@ -143,7 +143,7 @@ class Chat(object): :param path: path to an image file. :raises: ValueError if message can not be send/chat does not exist. - :returns: the resulting :class:`deltachat.chatting.Message` instance + :returns: the resulting :class:`deltachat.message.Message` instance """ if not os.path.exists(path): raise ValueError("path does not exist: {!r}".format(path)) @@ -179,7 +179,7 @@ class Chat(object): :param message: a :class:`Message` instance previously returned by :meth:`prepare_file`. :raises: ValueError if message can not be sent. - :returns: a :class:`deltachat.chatting.Message` instance with updated state + :returns: a :class:`deltachat.message.Message` instance with updated state """ msg_id = lib.dc_send_msg(self._dc_context, 0, message._dc_msg) if msg_id == 0: @@ -189,7 +189,7 @@ class Chat(object): def get_messages(self): """ return list of messages in this chat. - :returns: list of :class:`deltachat.chatting.Message` objects for this chat. + :returns: list of :class:`deltachat.message.Message` objects for this chat. """ dc_array = ffi.gc( lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0), diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index c7723758..4719e0b4 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -51,7 +51,7 @@ class Message(object): def get_state(self): """ get the message in/out state. - :returns: :class:`deltachat.chatting.MessageState` + :returns: :class:`deltachat.message.MessageState` """ return MessageState(self) @@ -89,7 +89,7 @@ class Message(object): def view_type(self): """the view type of this message. - :returns: a :class:`deltachat.chatting.MessageType` instance. + :returns: a :class:`deltachat.message.MessageType` instance. """ return MessageType(lib.dc_msg_get_viewtype(self._dc_msg)) diff --git a/python/tests/auditwheels.py b/python/tests/auditwheels.py new file mode 100644 index 00000000..eb4f5055 --- /dev/null +++ b/python/tests/auditwheels.py @@ -0,0 +1,13 @@ + +import os +import sys +import subprocess + + +if __name__ == "__main__": + assert len(sys.argv) == 2 + workspacedir = sys.argv[1] + for relpath in os.listdir(workspacedir): + if relpath.startswith("deltachat"): + p = os.path.join(workspacedir, relpath) + subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir]) diff --git a/python/tox.ini b/python/tox.ini index 52aac652..98158a9d 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -4,15 +4,25 @@ envlist = py27 py35 lint + auditwheels [testenv] -commands = pytest -rsXx {posargs:tests} +commands = + pytest -rsXx {posargs:tests} + pip wheel . -w {toxworkdir}/wheelhouse + passenv = TRAVIS deps = pytest pytest-faulthandler pdbpp +[testenv:auditwheels] +skipsdist = True +commands = + python tests/auditwheels.py {toxworkdir}/wheelhouse + + [testenv:lint] skipsdist = True usedevelop = True @@ -28,14 +38,13 @@ commands = rst-lint --encoding 'utf-8' README.rst [testenv:doc] -usedevelop = True deps = - sphinx + sphinx==1.8.5 breathe changedir = doc commands = - sphinx-build -b html . _build/html + sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html [pytest] diff --git a/python/wheelbuilder/Dockerfile b/python/wheelbuilder/Dockerfile deleted file mode 100644 index 08b55f97..00000000 --- a/python/wheelbuilder/Dockerfile +++ /dev/null @@ -1,123 +0,0 @@ -FROM quay.io/pypa/manylinux1_x86_64 - -# Configure ld.so/ldconfig and pkg-config -RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \ - echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf -ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig - -# Install meson and ninja -ENV NINJA_VERSION v1.8.2 -ENV NINJA_SHA256 d2fea9ff33b3ef353161ed906f260d565ca55b8ca0568fa07b1d2cab90a84a07 -RUN curl -L -o ninja-linux-$NINJA_VERSION.zip https://github.com/ninja-build/ninja/releases/download/${NINJA_VERSION}/ninja-linux.zip -RUN echo "${NINJA_SHA256} ninja-linux-${NINJA_VERSION}.zip" | sha256sum -c - -RUN unzip ninja-linux-${NINJA_VERSION}.zip -RUN mv ninja /usr/bin/ninja -RUN /opt/python/cp37-cp37m/bin/pip install meson -RUN cd /usr/bin && ln -s /opt/_internal/cpython-3.7.*/bin/meson - -# Install a recent zlib, needed by libetpan -ENV ZLIB_VERSION 1.2.11 -ENV ZLIB_SHA256 c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 -RUN curl -O https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz -RUN echo "${ZLIB_SHA256} zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c - -RUN tar xzf zlib-${ZLIB_VERSION}.tar.gz -RUN cd zlib-${ZLIB_VERSION} && ./configure -RUN cd zlib-${ZLIB_VERSION} && make -RUN cd zlib-${ZLIB_VERSION} && make install -RUN ldconfig -v - -# Install a recent Perl, needed to install OpenSSL -ENV PERL_VERSION 5.28.0 -ENV PERL_SHA256 7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8 -RUN curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz -RUN echo "${PERL_SHA256} perl-${PERL_VERSION}.tar.gz" | sha256sum -c - -RUN tar xzf perl-${PERL_VERSION}.tar.gz -RUN cd perl-${PERL_VERSION} && ./Configure -de -RUN cd perl-${PERL_VERSION} && make -RUN cd perl-${PERL_VERSION} && make install - -# Install OpenSSL -ENV OPENSSL_VERSION 1.1.1a -ENV OPENSSL_SHA256 fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41 -RUN curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c - -RUN tar xzf openssl-${OPENSSL_VERSION}.tar.gz -RUN cd openssl-${OPENSSL_VERSION} && \ - ./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local -RUN cd openssl-${OPENSSL_VERSION} && \ - sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \ - sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \ - sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile -RUN cd openssl-${OPENSSL_VERSION} && make depend -RUN cd openssl-${OPENSSL_VERSION} && make -RUN cd openssl-${OPENSSL_VERSION} && make install_sw install_ssldirs -RUN ldconfig -v - -# Install cyrus-sasl -ENV SASL_VERSION 2.1.27 -ENV SASL_SHA256 26866b1549b00ffd020f188a43c258017fa1c382b3ddadd8201536f72efb05d5 -RUN curl -O https://www.cyrusimap.org/releases/cyrus-sasl-${SASL_VERSION}.tar.gz -RUN echo "${SASL_SHA256} cyrus-sasl-${SASL_VERSION}.tar.gz" | sha256sum -c - -RUN tar zxf cyrus-sasl-${SASL_VERSION}.tar.gz -RUN cd cyrus-sasl-${SASL_VERSION} && \ - ./configure --disable-silent-rules \ - --disable-cmulocal \ - --disable-sample \ - --disable-obsolete_cram_attr \ - --disable-obsolete_digest_attr \ - --disable-staticdlopen \ - --disable-java \ - --disable-alwaystrue \ - --enable-checkapop \ - --enable-cram \ - --enable-digest \ - --enable-scram \ - --disable-otp \ - --disable-srp \ - --disable-srp-setpass \ - --disable-krb4 \ - --disable-gssapi \ - --disable-gss_mutexes \ - --disable-sia \ - --disable-auth-sasldb \ - --disable-httpform \ - --enable-plain \ - --enable-anon \ - --enable-login \ - --disable-ntlm \ - --disable-passdss \ - --disable-sql \ - --disable-ldapdb \ - --disable-macos-framework -RUN cd cyrus-sasl-${SASL_VERSION} && make -RUN cd cyrus-sasl-${SASL_VERSION} && make install -RUN ldconfig -v - -# Install libetpan -ENV ETPAN_VERSION 1.9.1 -ENV ETPAN_SHA256 f5e354ccf1014c6ee313ade1009b8a82f28043d2504655e388bb4c1328700fcd -RUN curl -L -o libetpan-${ETPAN_VERSION}.tar.gz \ - https://github.com/dinhviethoa/libetpan/archive/${ETPAN_VERSION}.tar.gz -RUN echo "${ETPAN_SHA256} libetpan-${ETPAN_VERSION}.tar.gz" | sha256sum -c - -RUN tar xzf libetpan-${ETPAN_VERSION}.tar.gz -RUN cd libetpan-${ETPAN_VERSION} && \ - ./autogen.sh && \ - ./configure --enable-ipv6 \ - --enable-iconv --disable-db \ - --with-openssl --with-sasl --with-zlib \ - --without-curl --without-expat -RUN cd libetpan-${ETPAN_VERSION} && make -RUN cd libetpan-${ETPAN_VERSION} && make install -RUN ldconfig -v - -# Install Rust nightly -RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y -ENV PATH=/root/.cargo/bin:$PATH -RUN rustc --version - -# Install RPGP from github -- this currently downloads 500MB -# see reported issue: https://github.com/dignifiedquire/rpgp/issues/30 -RUN true \ - && git clone https://github.com/dignifiedquire/rpgp.git \ - && cd rpgp/pgp-ffi \ - && make install diff --git a/python/wheelbuilder/build-wheels.sh b/python/wheelbuilder/build-wheels.sh deleted file mode 100755 index 657b209a..00000000 --- a/python/wheelbuilder/build-wheels.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e -x - -## Build the library -meson -Drpgp=true /builddir /io -pushd /builddir -ninja -ninja install -ldconfig -v -popd - -## Compile wheels -for PYBIN in /opt/python/*/bin; do - "${PYBIN}/pip" install cffi requests attrs six pytest - "${PYBIN}/pip" wheel /io/python -w wheelhouse/ -done - -## Bundle external shared libraries into the wheels -for whl in wheelhouse/deltachat*.whl; do - auditwheel repair "$whl" -w /io/python/wheelhouse/ -done - -## Install packages (and test) -for PYBIN in /opt/python/*/bin/; do - "${PYBIN}/pip" install deltachat --no-index -f /io/python/wheelhouse - # (cd "$HOME"; "${PYBIN}/pytest" /io/python/tests) -done