This page describes how we upgraded
from python 2.5.4 / django 1.1
to python 2.7.3 / django 1.3.1libraries on the web server.
This page describes the original installs.
We decided against using python virtualenv on the web servers.
Since we are using mod_wsgi, if we wanted to use different version of python,
we would have to recompile mod_wsgi against that version of python.
virtualenv does not help with this issue and it adds a bit of complexity so decided to go wiothout without it.
install python 2.7.3
| Code Block |
|---|
$ cd /usr/local/src
$sudo mkdir TARFILES
$sudo chown -R dev .
$ cd TARFILES
$
wget http://python.org/ftp/python/2.7.3/Python-2.7.3.tgz
# ... some $stdout
cd ..
$
tar xvfz TARFILES/Python-2.7.3.tgz $
cd Python-2.7.3
sudo $yum ./configureinstall -openssl-helpdevel $ ./configure --rpe
$ make
...
Python build finished, but the necessary bits to build these modules were not found:
_curses _curses_panel _sqlite3
_ssl bsddb185 bz2
dl gdbm imageop
readline sunaudiodev
To find the necessary bits, look in setup.py in detect_modules() for the module's name.
...
$ sudo yum install openssl-devel readline-devel ImageMagick sqllite-devel \
bzip2-devel gdbm-devel ncurses-devel curses-devel
...
# make yum stop complaining about incomplete transaction...
$ sudo yum install yum-utils
$ rpm -ql yum-i
$ sudo /usr/sbin/yum-complete-transaction
...
$ ./configure --prefix=/usr/local/python27 --enable-shared --with-threads
$ make clean
$ make
$ sudo make install
# make sure it runs
$readline-devel ImageMagick sqllite-devel \
bzip2-devel gdbm-devel ncurses-devel curses-devel
# ... lots of stdout
./configure --prefix=/usr/local/python27 --enable-shared --with-threads
make
sudo make install
# make sure it runs
/usr/local/python27/bin/python2.7
# Now, if you get errors about not being able to find libpython2.7.so.1.0,
# add this
# /usr/local/python27/lib
# to this file
# /etc/ld.so.conf.d/python2.7.conf
# then run:
sudo /sbin/ldconfig
# again - make sure it runs
/usr/local/python27/bin/python2.7
# if you want to reinstall, you can simply remove the old install this way
rm -rf /usr/local/python27
# and the install it again
|
upgrade mod_wsgi
| Code Block |
|---|
cd /usr/local/src/TARFILES
wget http://modwsgi.googlecode.com/files/mod_wsgi-3.3.tar.gz
cd ../
tar xvfz TARFILES/mod_wsgi-3.3.tar.gz
cd mod_wsgi-3.3
./configure --with-python=/usr/local/python27/bin/python
make
sudo make install |
upgrade psycopg2
| Code Block |
|---|
cd /usr/local/src/TARFILES/
wget http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz
cd ../
tar xvfz TARFILES/psycopg2-2.4.5.tar.gz
cd psycopg2-2.4.5/
sudo /usr/local/python27/bin/python2.7 setup.py install
/usr/local/python27/bin/python2.7
# echo should be as follows
# Python 2.7.3 (default, May 16 2012, 09:40:25)
# [GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2
# Type "help", "copyright", "credits" or "license" for more information.
# check your work
# >>> import psycopg2
# >>> psycopg2.__version__
# stdout should be
# '2.4.5 (dt dec pq3 ext)' #
# >>> exit()
|
upgrade proj.4
| Code Block |
|---|
# find and remove old proj libraries (?)
sudo find / -name "*proj*"
sudo find / -name "*proj*" | grep -v eas_ | grep -v Python-2 | grep -v usr\/local\/src | grep -v \.gz | grep -v django
# etc
cd /usr/local/lib
ls -l
sudo rm libproj.so.0.6.6
#
cd /usr/local/python27src/binTARFILES
$ ./python2.7
# Now, if you get errors about not being able to find libpython2.7.so.1.0,
# you probably need to create a new file
$ echo "/usr/local/python27/lib" > /etc/ld.so.conf.d/python2.7.conf
# and then run:
$ ldconfig
# if you want to reinstall, you can remove the old install this way
$ rm -rf /usr/local/python27
# and the install it again
|
upgrade mod_wsgi
| Code Block |
|---|
$ cd /usr/local/src/TARFILES
$ wget http://modwsgi.googlecode.com/files/mod_wsgi-3.3.tar.gz
$ cd ../
$ tar xvfz TARFILES/mod_wsgi-3.3.tar.gz
$ cd mod_wsgi-3.3
$ ./configure --with-python=/usr/local/python27/bin/python
$ make clean
$ make
$wget ftp://ftp.remotesensing.org/proj/proj-4.8.0.tar.gz
wget ftp://ftp.remotesensing.org/proj/proj-datumgrid-1.5.tar.gz
cd ../
tar xvfz TARFILES/proj-4.8.0.tar.gz
cd proj-4.8.0/nad
tar xvfz ../../TARFILES/proj-datumgrid-1.5.tar.gz
cd ..
./configure
make
sudo make install
|
upgrade geos
| Code Block |
|---|
# find old geos programs and libs and remove them
# do this iteratively with care
sudo find / -type f -name "*geos*" | grep -v src | grep -v geosilk | grep -v \.py
sudo find / -type f -name "*geos*" | grep -v src | grep -v geosilk | grep -v \.py | sudo xargs rm
# install new geos
cd /usr/local/src/TARFILES
wget http://download.osgeo.org/geos/geos-3.3.3.tar.bz2
cd ..
tar xvfj TARFILES/geos-3.3.3.tar.bz2
cd geos-3.3.3
# no python bindings
./configure
make
sudo make install |
upgrade
...
gdal
| Code Block |
|---|
$# cd /usr/local/src/TARFILES/
$ wget http://www.psycopg.org/psycopg/tarballs/PSYCOPG-2-4/psycopg2-2.4.5.tar.gz
$ cd ../
$ tar xvfz TARFILES/psycopg2-2.4.5.tar.gz
$ cd psycopg2-2.4.5/
$ sudo /usr/local/python27/bin/python2.7 setup.py install
$ /usr/local/python27/bin/python2.7
Python 2.7.3 (default, May 16 2012, 09:40:25)
[GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import psycopg2
>>> exit()
$ |
upgrade proj.4
| Code Block |
|---|
$ cd /usr/local/src/TARFILES
$ wget ftp://ftp.remotesensing.org/proj/proj-4.8.0.tar.gz
$ wget ftp://ftp.remotesensing.org/proj/proj-datumgrid-1.5.tar.gz
$ cd ../
$ tar xvfz TARFILES/proj-4.8.0.tar.gz
$ cd proj-4.8.0/nad
$ tar xvfz ../../TARFILES/proj-datumgrid-1.5.tar.gz
$ cd ..
$ ./configure
$ make
$ sudo make install
# find and remove old proj libraries (?)
$ sudo find / -name "*proj*"
$ cd /usr/local/lib
$ ls -l
$ sudo rm libproj.so.0.6.6
$ |
upgrade geos
...
find old gdal and remove
sudo rm -rf /usr/local/share/gdal/
# inspect output of this find...
sudo find / -name "*gdal*" | grep -v 'src\|python2.5'
# then pipe to xargs rm
sudo find / -name "*gdal*" | grep -v 'src\|python2.5' | sudo xargs rm
#
cd /usr/local/src/TARFILES
wget http://download.osgeo.org/gdal/gdal-1.9.0.tar.gz
cd ../
tar xvfz TARFILES/gdal-1.9.0.tar.gz
cd gdal-1.9.0/
# set your PATH properly so we get the right version of python
export PATH=/usr/local/python27/bin:$PATH
./configure --with-python
make
sudo PATH=/usr/local/python27/bin:$PATH make install
|
upgrade django
| Code Block |
|---|
cd /usr/local/src/TARFILES
wget http://www.djangoproject.com/m/releases/1.3/Django-1.3.1.tar.gz
cd ../
tar xvfz TARFILES/Django-1.3.1.tar.gz
cd Django-1.3.1/
sudo /usr/local/python27/bin/python2.7 setup.py install
|
check your work
To make sure everything is working as expected, use set_eas_mode.sh then visit the root URL.
| Code Block |
|---|
$ sudo ./set_eas_mode.sh SHOW_ENV
|
Then visit the URL of the web server and you should see something like this in the browser:
| Code Block |
|---|
python version info:
2.7.3 (default, May 16 2012, 09:40:25)
[GCC 4.1.2 20080704 (Red Hat 4.1.2-48)]
django version info:
(1, 3, 1, 'final', 0)
psycopg2 version info:
2.4.5 (dt dec pq3 ext)
mod_wsgi version info:
(3, 3)
gdal version info:
(1, 9, 0)
geos version info:
{'major': '3', 'subminor': '3', 'version': '3.3.3', 'capi_version': '1.7.4', 'release_candidate': None, 'minor': '3'} |
If you see an internal server you probably just need to ldconfig...
| Code Block |
|---|
sudo /sbin/ldconfig |
OBSOLETE - install virtualenv 2.7
| Code Block |
|---|
$ wget http://python-distribute.org/distribute_setup.py
$ sudo /usr/local/python27/bin/python distribute_setup.py
$ wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py
$ sudo /usr/local/python27/bin/python get-pip.py
$ sudo /usr/local/python27/bin/pip install virtualenv
$ sudo ln -s /usr/local/python27/bin/virtualenv /usr/local/bin/virtualenv-2.7 |
OBSOLETE - setting up a virtualenv
| Code Block |
|---|
$ mkdir virtualenvs
$ cd virtualenvs
$ virtualenv-2.7 --no-site-packages --python=/usr/local/python27/bin/python python27_paul
$ . ~/virtualenvs/python27_paul/bin/activate
(python27)$ deactivate
$ |
OBSOLETE - install gdal 1.8.1 into a virtualenv
| Code Block |
|---|
$ . ~/virtualenvs/python27/bin/activate
(python27_paul)$ cd /usr/local/src/gdal-1.8.1/
(python27_paul)$ ./configure --prefix=/home/dev/virtualenvs/python27_paul
(python27_paul)$ make && make install
(python27_paul)$ pip install --no-install GDAL==1.8.1
(python27_paul)$ vim ~/virtualenvs/python27_paul/build/GDAL/setup.py
#
# replace
# extra_link_args = []
# with
# extra_link_args = ['-Wl,-R/home/dev/virtualenvs/python27_paul/lib']
#
(python27_paul)$ cd virtualenvs/python27_paul/build/GDAL
(python27_paul)$ python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27_paul/lib --include-dirs=/home/dev/virtualenvs/python27_paul/include install
#
# confirm linked library is correct
#
$ ldd /home/dev/virtualenvs/python27/lib/python2.7/site-packages/GDAL-1.8.1-py2.7-linux-x86_64.egg/osgeo/_gdal.so
#
# should be pointing to the correct version of the libs
#
# if you start python in the GDAL dir, and try to import gdal you get an error
#
(python27_paul)$ pwd
/home/dev/virtualenvs/python27_paul/build/GDAL
(python27_paul)$ python
Python 2.7.3 (default, May 9 2012, 14:28:18) [GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] ...
>>> import gdal
ImportError: No module named _gdal
>>> exit()
#
# therefore do not start python in build/GDAL!
#
(python27_paul)$ cd ~
(python27_paul)$ pwd
/home/dev
(python27_paul)$ python
...
>>> import gdal
>>> exit()
(python27_paul)$ deactivate
$
|
OBSOLETE - email from Jeff Frost
| Code Block |
|---|
We win!
Here's how I got it to work.
Recreated our virtualenv with --no-site-packages just to be clean (I don't think that was necessary, but just in case):
rm -rf ~/virtualenvs/python27
virtualenv-2.6 --no-site-packages --python=/usr/local/python27/bin/python python27 . ~/virtualenvs/python27/bin/activate
As you recall, we installed gdal-1.8.1 in the virtualenv like so:
./configure --prefix=/home/dev/virtualenvs/python27 --with-python make && make install
Well, this time I installed it without the --with-python option:
./configure --prefix=/home/dev/virtualenvs/python27
make && make install
EDIT: I decided to go back and test after creating a pristine virtualenv, and you don't have to do anything but:
pip-install GDAL==1.8.1
(python27)[dev@easdevweb
python27]$ python Python 2.7.3 (default, May 9 2012, 14:28:18) [GCC
4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2 Type "help", "copyright",
"credits" or "license" for more information.
>>> import gdal
>>>
So,
I think what happened is that allowing access to the site-packages also
allowed access to the other rpaths and thus linked against the wrong
gdal at runtime.
Read on for more interesting info though:
So, how to change the rpath for the python module?
pip install --no-install GDAL==1.8.1
cd build/GDAL/
vim setup.py
Just gotta add some extra_link_args. Here's a diff, so you can see what I
changed:
(python27)[dev@easdevweb GDAL]$ diff setup.py setup.py.orig
165c165
< extra_link_args = ['-Wl,-R/home/dev/virtualenvs/python27/lib']
---
> extra_link_args = []
python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27/lib
--include-dirs=/home/dev/virtualenvs/python27/include
and voila:
(python27)[dev@easdevweb GDAL]$ ldd
build/lib.linux-x86_64-2.7/osgeo/_gdal_array.so
libgdal.so.1 => /home/dev/virtualenvs/python27/lib/libgdal.so.1
(0x00002ae0b273c000)
libstdc++.so.6 => /usr/lib64/libstdc++.so.6 (0x00002ae0b329e000)
libm.so.6 => /lib64/libm.so.6 (0x00002ae0b359e000)
libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00002ae0b3822000)
libpthread.so.0 => /lib64/libpthread.so.0 (0x00002ae0b3a30000)
libc.so.6 => /lib64/libc.so.6 (0x00002ae0b3c4b000)
libgeos_c.so.1 => /usr/local/lib/libgeos_c.so.1 (0x00002ae0b3fa3000)
libsqlite3.so.0 => /usr/lib64/libsqlite3.so.0 (0x00002ae0b41ba000)
libexpat.so.0 => /lib64/libexpat.so.0 (0x00002ae0b4415000)
libpq.so.4 => /usr/lib64/libpq.so.4 (0x00002ae0b4638000)
libz.so.1 => /usr/lib64/libz.so.1 (0x00002ae0b485a000)
librt.so.1 => /lib64/librt.so.1 (0x00002ae0b4a6e000)
libdl.so.2 => /lib64/libdl.so.2 (0x00002ae0b4c78000)
/lib64/ld-linux-x86-64.so.2 (0x00000039f2e00000)
libgeos-3.1.1.so => /usr/local/lib/libgeos-3.1.1.so (0x00002ae0b4e7c000)
libssl.so.6 => /lib64/libssl.so.6 (0x00002ae0b51dd000)
libcrypto.so.6 => /lib64/libcrypto.so.6 (0x00002ae0b5429000)
libkrb5.so.3 => /usr/lib64/libkrb5.so.3 (0x00002ae0b577a000)
libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00002ae0b5a10000)
libresolv.so.2 => /lib64/libresolv.so.2 (0x00002ae0b5c48000)
libnsl.so.1 => /lib64/libnsl.so.1 (0x00002ae0b5e5d000)
libgssapi_krb5.so.2 => /usr/lib64/libgssapi_krb5.so.2 (0x00002ae0b6076000)
libcom_err.so.2 => /lib64/libcom_err.so.2 (0x00002ae0b62a4000)
libk5crypto.so.3 => /usr/lib64/libk5crypto.so.3 (0x00002ae0b64a6000)
libkrb5support.so.0 => /usr/lib64/libkrb5support.so.0 (0x00002ae0b66cc000)
libkeyutils.so.1 => /lib64/libkeyutils.so.1 (0x00002ae0b68d4000)
libselinux.so.1 => /lib64/libselinux.so.1 (0x00002ae0b6ad7000)
libsepol.so.1 => /lib64/libsepol.so.1 (0x00002ae0b6cef000)
python setup.py build_ext --gdal-config=gdal-config --library-dirs=/home/dev/virtualenvs/python27/lib --include-dirs=/home/dev/virtualenvs/python27/include install
(python27)[dev@easdevweb
site-packages]$ python Python 2.7.3 (default, May 9 2012, 14:28:18)
[GCC 4.1.2 20080704 (Red Hat 4.1.2-48)] on linux2 Type "help",
"copyright", "credits" or "license" for more information.
>>> import gdal
>>>
--
Jeff Frost <jeff@pgexperts.com>
CTO, PostgreSQL Experts, Inc.
Phone: 1-888-PG-EXPRT x506
FAX: 415-762-5122
http://www.pgexperts.com/ |
...