Merge pull request #156 from mapnik/master-mason

Get master passing on travis

Author: springmeyer

.travis.yml

@@ -1,17 +1,10 @@
 language: generic
 
-git:
-  submodules: true
-  depth: 10
-
-addons:
-  postgresql: "9.4"
-
 matrix:
   include:
     - os: linux
       sudo: false
-      compiler: ": clang"
+      compiler: clang
       # note: only using ccache for CC is intentional here to
       # workaround an odd bug in distutils that manifests when only `ccache` is used to link
       # because distutils also has a bug whereby CC is used to compile instead of CXX, this works :)
@@ -20,11 +13,10 @@ matrix:
         apt:
           sources: [ 'ubuntu-toolchain-r-test']
           packages: [ 'libstdc++-5-dev', 'gdb', 'apport']
-    # disabled until I have time to fix https://github.com/mapnik/python-mapnik/issues/95
-    #- os: osx
-    #  osx_image: xcode7.3 # upgrades clang from 6 -> 7
-    #  compiler: clang
-    #  env: JOBS=4
+    - os: osx
+      osx_image: xcode8.2
+      compiler: clang
+      env: JOBS=4
 
 cache:
   directories:
@@ -56,22 +48,13 @@ before_install:
  - |
    if [[ $(uname -s) == 'Linux' ]]; then
      export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++-3.9 ')");
-     ./.mason/mason install clang++ 3.9.1
-     export PATH=$(./.mason/mason prefix clang++ 3.9.1)/bin:${PATH}
+     mason install clang++ 3.9.1
+     export PATH=$(mason prefix clang++ 3.9.1)/bin:${PATH}
      which clang++
    else
      sudo easy_install pip;
-     brew info postgres;
-     export DATA_PATH=$(brew --prefix)/var/postgres;
-     rm -rf ${DATA_PATH};
-     initdb ${DATA_PATH} -E utf8;
-     pg_ctl -w start -l postgres.log --pgdata ${DATA_PATH};
-     cat postgres.log;
-     createuser -s postgres;
      export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++ ')");
    fi
- - psql -c 'create database template_postgis;' -U postgres
- - psql -c 'create extension postgis;' -d template_postgis -U postgres
  - pip install --upgrade --user nose
  - pip install --upgrade --user wheel
  - pip install --upgrade --user twine
@@ -84,15 +67,15 @@ install:
  - python setup.py install --prefix ${PYTHONUSERBASE}
 
 before_script:
- - ulimit -c unlimited -S
- - RESULT=0
- - python test/run_tests.py || RESULT=$?
- - ls -l .
- - for i in $(find ./ -maxdepth 1 -name 'core*' -print); do gdb $(which python) $i -ex "thread apply all bt" -ex "set pagination 0" -batch; done;
- - if [[ ${RESULT} != 0 ]]; then exit $RESULT; fi
+ # start postgres/postgis
+ - source mason-config.env
+ - ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
 
 script:
+ - python test/run_tests.py
  - python test/visual.py -q
+ # stop postgres
+ - ./mason_packages/.link/bin/pg_ctl -w stop
  - |
    if [[ ${COMMIT_MESSAGE} =~ "[publish]" ]]; then
      python setup.py bdist_wheel

bootstrap.sh

@@ -14,42 +14,61 @@ function install() {
 ICU_VERSION="57.1"
 
 function install_mason_deps() {
-    install mapnik 333ef9f
-    install jpeg_turbo 1.5.1 libjpeg
-    install libpng 1.6.28 libpng
-    install libtiff 4.0.7 libtiff
+    install mapnik df0bbe4
+    install jpeg_turbo 1.5.1
+    install libpng 1.6.28
+    install libtiff 4.0.7
     install libpq 9.6.2
-    install sqlite 3.17.0 libsqlite3
-    install expat 2.2.0 libexpat
+    install sqlite 3.17.0
+    install expat 2.2.0
     install icu ${ICU_VERSION}
-    install proj 4.9.3 libproj
-    install pixman 0.34.0 libpixman-1
-    install cairo 1.14.8 libcairo
-    install webp 0.6.0 libwebp
-    install libgdal 2.1.3 libgdal
+    install proj 4.9.3
+    install pixman 0.34.0
+    install cairo 1.14.8
+    install webp 0.6.0
+    install libgdal 2.1.3
     install boost 1.63.0
     install boost_libsystem 1.63.0
     install boost_libfilesystem 1.63.0
     install boost_libprogram_options 1.63.0
     install boost_libregex_icu57 1.63.0
-    install freetype 2.7.1 libfreetype
-    install harfbuzz 1.4.2-ft libharfbuzz
+    install freetype 2.7.1
+    install harfbuzz 1.4.2-ft
     # deps needed by python-mapnik (not mapnik core)
     install boost_libthread 1.63.0
     install boost_libpython 1.63.0
+    install postgis 2.3.2-1
 }
 
 function setup_runtime_settings() {
     local MASON_LINKED_ABS=$(pwd)/mason_packages/.link
-    export PROJ_LIB=${MASON_LINKED_ABS}/share/proj
-    export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}
-    export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal
-    if [[ $(uname -s) == 'Darwin' ]]; then
-        export DYLD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${DYLD_LIBRARY_PATH}
-    else
-        export LD_LIBRARY_PATH=$(pwd)/mason_packages/.link/lib:${LD_LIBRARY_PATH}
-    fi
-    export PATH=$(pwd)/mason_packages/.link/bin:${PATH}
+    echo "export PROJ_LIB=${MASON_LINKED_ABS}/share/proj" > mason-config.env
+    echo "export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}" >> mason-config.env
+    echo "export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal" >> mason-config.env
+    echo "export PATH=$(pwd)/mason_packages/.link/bin:${PATH}" >> mason-config.env
+    echo "export PGTEMP_DIR=$(pwd)/local-tmp" >> mason-config.env
+    echo "export PGDATA=$(pwd)/local-postgres" >> mason-config.env
+    echo "export PGHOST=$(pwd)/local-unix-socket" >> mason-config.env
+    echo "export PGPORT=1111" >> mason-config.env
+
+    source mason-config.env
+    rm -rf ${PGHOST}
+    mkdir -p ${PGHOST}
+    rm -rf ${PGDATA}
+    mkdir -p ${PGDATA}
+    rm -rf ${PGTEMP_DIR}
+    mkdir -p ${PGTEMP_DIR}
+    ./mason_packages/.link/bin/initdb
+    sleep 2
+    ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log &
+    sleep 2
+    ./mason_packages/.link/bin/createdb template_postgis -T postgres
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE TABLESPACE temp_disk LOCATION '${PGTEMP_DIR}';"
+    ./mason_packages/.link/bin/psql template_postgis -c "SET temp_tablespaces TO 'temp_disk';"
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;"
+    ./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION postgis;"
+    ./mason_packages/.link/bin/psql template_postgis -c "SELECT PostGIS_Full_Version();"
+    ./mason_packages/.link/bin/pg_ctl -w stop
 }
 
 function main() {
@@ -59,10 +78,10 @@ function main() {
     setup_runtime_settings
     echo "Ready, now run:"
     echo ""
-    echo "    python setup.py test"
+    echo "    make test"
 }
 
 main
 
 set +eu
-set +o pipefail
+set +o pipefail

scripts/setup_mason.sh

@@ -4,18 +4,13 @@ set -eu
 set -o pipefail
 
 # we pin the mason version to avoid changes in mason breaking builds
-MASON_VERSION="68a9498"
+MASON_VERSION="3870d9c"
 
 function setup_mason() {
-    if [[ ! -d ./.mason ]]; then
-        git clone https://github.com/mapbox/mason.git ./.mason
-        (cd ./.mason && git checkout ${MASON_VERSION})
-    else
-        echo "Updating to latest mason"
-        (cd ./.mason && git fetch && git checkout ${MASON_VERSION})
-    fi
+    mkdir -p ./mason
+    curl -sSfL https://github.com/mapbox/mason/archive/${MASON_VERSION}.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason
     export MASON_HOME=$(pwd)/mason_packages/.link
-    export PATH=$(pwd)/.mason:$PATH
+    export PATH=$(pwd)/mason:${PATH}
     export CXX=${CXX:-clang++}
     export CC=${CC:-clang}
 }

setup.py

@@ -120,14 +120,10 @@ def run(self):
 '-lmapnik-json',
 ] + ['-l%s' % i for i in get_boost_library_names()])
 
-# Dynamically make the mapnik/paths.py file if it doesn't exist.
-if os.path.isfile('mapnik/paths.py'):
-    create_paths = False
-else:
-    create_paths = True
-    f_paths = open('mapnik/paths.py', 'w')
-    f_paths.write('import os\n')
-    f_paths.write('\n')
+# Dynamically make the mapnik/paths.py file
+f_paths = open('mapnik/paths.py', 'w')
+f_paths.write('import os\n')
+f_paths.write('\n')
 
 input_plugin_path = check_output([mapnik_config, '--input-plugins'])
 font_path = check_output([mapnik_config, '--fonts'])
@@ -147,30 +143,32 @@ def run(self):
     input_plugin_files = os.listdir(input_plugin_path)
     input_plugin_files = [os.path.join(
         input_plugin_path, f) for f in input_plugin_files]
-    if not os.path.exists(os.path.join('mapnik', 'plugins', 'input')):
-        os.makedirs(os.path.join('mapnik', 'plugins', 'input'))
+    if not os.path.exists(os.path.join('mapnik', 'lib', 'mapnik', 'input')):
+        os.makedirs(os.path.join('mapnik', 'lib', 'mapnik', 'input'))
     for f in input_plugin_files:
         try:
             shutil.copyfile(f, os.path.join(
-                'mapnik', 'plugins', 'input', os.path.basename(f)))
+                'mapnik', 'lib', 'mapnik', 'input', os.path.basename(f)))
         except shutil.Error:
             pass
     font_files = os.listdir(font_path)
     font_files = [os.path.join(font_path, f) for f in font_files]
-    if not os.path.exists(os.path.join('mapnik', 'plugins', 'fonts')):
-        os.makedirs(os.path.join('mapnik', 'plugins', 'fonts'))
+    if not os.path.exists(os.path.join('mapnik', 'lib', 'mapnik', 'fonts')):
+        os.makedirs(os.path.join('mapnik', 'lib', 'mapnik', 'fonts'))
     for f in font_files:
         try:
             shutil.copyfile(f, os.path.join(
-                'mapnik', 'plugins', 'fonts', os.path.basename(f)))
+                'mapnik', 'lib', 'mapnik', 'fonts', os.path.basename(f)))
         except shutil.Error:
             pass
-    if create_paths:
-        f_paths.write(
-            'mapniklibpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "plugins")\n')
-        f_paths.write("inputpluginspath = os.path.join(mapniklibpath,'input')\n")
-        f_paths.write("fontscollectionpath = os.path.join(mapniklibpath,'fonts')\n")
-elif create_paths:
+    f_paths.write(
+        'mapniklibpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib")\n')
+    f_paths.write("inputpluginspath = os.path.join(mapniklibpath, 'mapnik', 'input')\n")
+    f_paths.write("fontscollectionpath = os.path.join(mapniklibpath, 'mapnik', 'fonts')\n")
+    f_paths.write(
+        "__all__ = [mapniklibpath,inputpluginspath,fontscollectionpath]\n")
+    f_paths.close()
+else:
     if os.environ.get('LIB_DIR_NAME'):
         mapnik_lib_path = lib_path + os.environ.get('LIB_DIR_NAME')
     else:
@@ -181,8 +179,6 @@ def run(self):
         "inputpluginspath = '{path}'\n".format(path=input_plugin_path))
     f_paths.write(
         "fontscollectionpath = '{path}'\n".format(path=font_path))
-
-if create_paths:
     f_paths.write(
         "__all__ = [mapniklibpath,inputpluginspath,fontscollectionpath]\n")
     f_paths.close()
@@ -269,7 +265,7 @@ def run(self):
         'nose',
     ],
     package_data={
-        'mapnik': ['lib/*', 'plugins/*/*', 'share/*/*'],
+        'mapnik': ['lib/*.*', 'lib/*/*/*', 'share/*/*'],
     },
     test_suite='nose.collector',
     cmdclass={