Release 1.1.1 - Upgrade Web Server (archive)

Owned by Paul McCullough (Unlicensed)
Last updated: Jul 27, 2018 by Samuel Valdez (Unlicensed)Version comment
5 min read

This page describes how we upgraded libraries on the web server.

This page describes the original installs.

We decided against using python virtualenv on the web servers.
Since we are using mod_wsgi, if we wanted to use different version of python,
we would have to recompile mod_wsgi against that version of python.
virtualenv does not help with this issue and it adds a bit of complexity so decided to go without it.


install python 2.7.3

cd /usr/local/src
sudo mkdir TARFILES
sudo chown -R dev .
cd TARFILES
wget http://python.org/ftp/python/2.7.3/Python-2.7.3.tgz
# ... some stdout
cd ..
tar xvfz TARFILES/Python-2.7.3.tgz 
cd Python-2.7.3
sudo yum install openssl-devel readline-devel ImageMagick sqllite-devel \
  bzip2-devel gdbm-devel ncurses-devel curses-devel
# ... lots of stdout
./configure --prefix=/usr/local/python27 --enable-shared --with-threads
make
sudo make install

# make sure it runs
/usr/local/python27/bin/python2.7

# Now, if you get errors about not being able to find libpython2.7.so.1.0,
# add this 
#    /usr/local/python27/lib
# to this file
#    /etc/ld.so.conf.d/python2.7.conf
# then run:
sudo /sbin/ldconfig

# again - make sure it runs
/usr/local/python27/bin/python2.7

# if you want to reinstall, you can simply remove the old install this way
rm -rf /usr/local/python27
# and the install it again
 

upgrade mod_wsgi

cd /usr/local/src/TARFILES
wget http://modwsgi.googlecode.com/files/mod_wsgi-3.3.tar.gz
cd ../
tar xvfz TARFILES/mod_wsgi-3.3.tar.gz
cd mod_wsgi-3.3
./configure --with-python=/usr/local/python27/bin/python
make
sudo make install

upgrade psycopg2

cd /usr/local/src/TARFILES/
wget http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz
cd ../
tar xvfz TARFILES/psycopg2-2.4.5.tar.gz
cd psycopg2-2.4.5/
sudo /usr/local/python27/bin/python2.7 setup.py install
/usr/local/python27/bin/python2.7
# echo should be as follows
# Python 2.7.3 (default, May 16 2012, 09:40:25)
# [GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2
# Type "help", "copyright", "credits" or "license" for more information.

# check your work
# >>> import psycopg2
# >>> psycopg2.__version__
# stdout should be
# '2.4.5 (dt dec pq3 ext)' # 
# >>> exit()

upgrade proj.4

# find and remove old proj libraries (?)
sudo find / -name "*proj*"
sudo find / -name "*proj*" | grep -v eas_ | grep -v Python-2 | grep -v usr\/local\/src | grep -v \.gz | grep -v django
# etc
cd /usr/local/lib
ls -l
sudo rm libproj.so.0.6.6
#
cd /usr/local/src/TARFILES
wget ftp://ftp.remotesensing.org/proj/proj-4.8.0.tar.gz
wget ftp://ftp.remotesensing.org/proj/proj-datumgrid-1.5.tar.gz
cd ../
tar xvfz TARFILES/proj-4.8.0.tar.gz
cd proj-4.8.0/nad
tar xvfz ../../TARFILES/proj-datumgrid-1.5.tar.gz
cd ..
./configure
make
sudo make install

upgrade geos

# find old geos programs and libs and remove them
# do this iteratively with care
sudo find / -type f -name "*geos*" | grep -v src | grep -v geosilk | grep -v \.py
sudo find / -type f -name "*geos*" | grep -v src | grep -v geosilk | grep -v \.py | sudo xargs rm

# install new geos
cd /usr/local/src/TARFILES
wget http://download.osgeo.org/geos/geos-3.3.3.tar.bz2
cd ..
tar xvfj TARFILES/geos-3.3.3.tar.bz2
cd geos-3.3.3
# no python bindings
./configure
make
sudo make install

upgrade gdal

# find old gdal and remove
sudo rm -rf /usr/local/share/gdal/
# inspect output of this find...
sudo find / -name "*gdal*" | grep -v 'src\|python2.5'
# then pipe to xargs rm
sudo find / -name "*gdal*" | grep -v 'src\|python2.5' | sudo xargs rm
#
cd /usr/local/src/TARFILES
wget http://download.osgeo.org/gdal/gdal-1.9.0.tar.gz
cd ../
tar xvfz TARFILES/gdal-1.9.0.tar.gz
cd gdal-1.9.0/
# set your PATH properly so we get the right version of python
export PATH=/usr/local/python27/bin:$PATH
./configure --with-python
make
sudo PATH=/usr/local/python27/bin:$PATH make install

upgrade django

cd /usr/local/src/TARFILES
wget http://www.djangoproject.com/m/releases/1.3/Django-1.3.1.tar.gz
cd ../
tar xvfz TARFILES/Django-1.3.1.tar.gz
cd Django-1.3.1/
sudo /usr/local/python27/bin/python2.7 setup.py install

check your work

To make sure everything is working as expected, use set_eas_mode.sh then visit the root URL.

$ sudo ./set_eas_mode.sh SHOW_ENV

Then visit the URL of the web server and you should see something like this in the browser:

python version info:
2.7.3 (default, May 16 2012, 09:40:25) 
[GCC 4.1.2 20080704 (Red Hat 4.1.2-48)]

django version info:
(1, 3, 1, 'final', 0)

psycopg2 version info:
2.4.5 (dt dec pq3 ext)

mod_wsgi version info:
(3, 3)

gdal version info:
(1, 9, 0)

geos version info:
{'major': '3', 'subminor': '3', 'version': '3.3.3', 'capi_version': '1.7.4', 'release_candidate': None, 'minor': '3'}

If you see an internal server you probably just need to ldconfig...

sudo /sbin/ldconfig


OBSOLETE - install virtualenv 2.7

$ wget http://python-distribute.org/distribute_setup.py
$ sudo /usr/local/python27/bin/python distribute_setup.py
$ wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py
$ sudo /usr/local/python27/bin/python get-pip.py
$ sudo /usr/local/python27/bin/pip install virtualenv
$ sudo ln -s /usr/local/python27/bin/virtualenv /usr/local/bin/virtualenv-2.7 

OBSOLETE - setting up a virtualenv

$ mkdir virtualenvs
$ cd virtualenvs
$ virtualenv-2.7 --no-site-packages --python=/usr/local/python27/bin/python python27_paul
$ . ~/virtualenvs/python27_paul/bin/activate
(python27)$ deactivate
$

OBSOLETE - install gdal 1.8.1 into a virtualenv

$ . ~/virtualenvs/python27/bin/activate
(python27_paul)$ cd /usr/local/src/gdal-1.8.1/
(python27_paul)$ ./configure --prefix=/home/dev/virtualenvs/python27_paul
(python27_paul)$ make && make install
(python27_paul)$ pip install --no-install GDAL==1.8.1
(python27_paul)$ vim ~/virtualenvs/python27_paul/build/GDAL/setup.py
#
# replace
#   extra_link_args = []
# with 
#   extra_link_args = ['-Wl,-R/home/dev/virtualenvs/python27_paul/lib']
#
(python27_paul)$ cd virtualenvs/python27_paul/build/GDAL

(python27_paul)$ python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27_paul/lib --include-dirs=/home/dev/virtualenvs/python27_paul/include install
#
# confirm linked library is correct
#
$ ldd /home/dev/virtualenvs/python27/lib/python2.7/site-packages/GDAL-1.8.1-py2.7-linux-x86_64.egg/osgeo/_gdal.so
#
# should be pointing to the correct version of the libs
#
# if you start python in the GDAL dir, and try to import gdal you get an error
#
(python27_paul)$ pwd
/home/dev/virtualenvs/python27_paul/build/GDAL
(python27_paul)$ python 
Python 2.7.3 (default, May  9 2012, 14:28:18) [GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] ...
>>> import gdal
ImportError: No module named _gdal
>>> exit()
#
# therefore do not start python in build/GDAL!
#
(python27_paul)$ cd ~
(python27_paul)$ pwd
/home/dev
(python27_paul)$ python
...
>>> import gdal
>>> exit()
(python27_paul)$ deactivate
$

OBSOLETE - email from Jeff Frost

We win! 
Here's how I got it to work.

Recreated our virtualenv with --no-site-packages just to be clean (I don't think that was necessary, but just in case):
rm -rf ~/virtualenvs/python27
virtualenv-2.6 --no-site-packages --python=/usr/local/python27/bin/python python27 . ~/virtualenvs/python27/bin/activate

As you recall, we installed gdal-1.8.1 in the virtualenv like so:

./configure --prefix=/home/dev/virtualenvs/python27 --with-python make && make install

Well, this time I installed it without the --with-python option:
./configure --prefix=/home/dev/virtualenvs/python27
make && make install

EDIT: I decided to go back and test after creating a pristine virtualenv, and you don't have to do anything but:
pip-install GDAL==1.8.1

(python27)[dev@easdevweb
 python27]$ python Python 2.7.3 (default, May  9 2012, 14:28:18) [GCC 
4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2 Type "help", "copyright", 
"credits" or "license" for more information.
>>> import gdal
>>> 

So,
 I think what happened is that allowing access to the site-packages also
 allowed access to the other rpaths and thus linked against the wrong 
gdal at runtime.

Read on for more interesting info though:

So, how to change the rpath for the python module?

pip install --no-install GDAL==1.8.1
cd build/GDAL/
vim setup.py

Just gotta add some extra_link_args. Here's  a diff, so you can see what I
changed:

(python27)[dev@easdevweb GDAL]$ diff setup.py setup.py.orig
165c165
< extra_link_args = ['-Wl,-R/home/dev/virtualenvs/python27/lib']
---
> extra_link_args = []

python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27/lib
--include-dirs=/home/dev/virtualenvs/python27/include

and voila:

(python27)[dev@easdevweb GDAL]$ ldd
build/lib.linux-x86_64-2.7/osgeo/_gdal_array.so                                                      
libgdal.so.1 => /home/dev/virtualenvs/python27/lib/libgdal.so.1
(0x00002ae0b273c000)
        libstdc++.so.6 => /usr/lib64/libstdc++.so.6 (0x00002ae0b329e000)
        libm.so.6 => /lib64/libm.so.6 (0x00002ae0b359e000)
        libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00002ae0b3822000)
        libpthread.so.0 => /lib64/libpthread.so.0 (0x00002ae0b3a30000)
        libc.so.6 => /lib64/libc.so.6 (0x00002ae0b3c4b000)
        libgeos_c.so.1 => /usr/local/lib/libgeos_c.so.1 (0x00002ae0b3fa3000)
        libsqlite3.so.0 => /usr/lib64/libsqlite3.so.0 (0x00002ae0b41ba000)
        libexpat.so.0 => /lib64/libexpat.so.0 (0x00002ae0b4415000)
        libpq.so.4 => /usr/lib64/libpq.so.4 (0x00002ae0b4638000)
        libz.so.1 => /usr/lib64/libz.so.1 (0x00002ae0b485a000)
        librt.so.1 => /lib64/librt.so.1 (0x00002ae0b4a6e000)
        libdl.so.2 => /lib64/libdl.so.2 (0x00002ae0b4c78000)
        /lib64/ld-linux-x86-64.so.2 (0x00000039f2e00000)
        libgeos-3.1.1.so => /usr/local/lib/libgeos-3.1.1.so (0x00002ae0b4e7c000)
        libssl.so.6 => /lib64/libssl.so.6 (0x00002ae0b51dd000)
        libcrypto.so.6 => /lib64/libcrypto.so.6 (0x00002ae0b5429000)
        libkrb5.so.3 => /usr/lib64/libkrb5.so.3 (0x00002ae0b577a000)
        libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00002ae0b5a10000)
        libresolv.so.2 => /lib64/libresolv.so.2 (0x00002ae0b5c48000)
        libnsl.so.1 => /lib64/libnsl.so.1 (0x00002ae0b5e5d000)
        libgssapi_krb5.so.2 => /usr/lib64/libgssapi_krb5.so.2 (0x00002ae0b6076000)
        libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00002ae0b62a4000)
        libk5crypto.so.3 => /usr/lib64/libk5crypto.so.3 (0x00002ae0b64a6000)
        libkrb5support.so.0 => /usr/lib64/libkrb5support.so.0 (0x00002ae0b66cc000)
        libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00002ae0b68d4000)
        libselinux.so.1 => /lib64/libselinux.so.1 (0x00002ae0b6ad7000)
        libsepol.so.1 => /lib64/libsepol.so.1 (0x00002ae0b6cef000)

python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27/lib --include-dirs=/home/dev/virtualenvs/python27/include install


(python27)[dev@easdevweb
 site-packages]$ python Python 2.7.3 (default, May  9 2012, 14:28:18) 
[GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2 Type "help", 
"copyright", "credits" or "license" for more information.
>>> import gdal
>>> 

--
Jeff Frost <jeff@pgexperts.com>
CTO, PostgreSQL Experts, Inc.
Phone: 1-888-PG-EXPRT x506
FAX: 415-762-5122
http://www.pgexperts.com/