Fix pip install settings

There were some inconsistencies with pip install instructions
thoughout Kolla. We fix those here.

Additionally, we fix the virtualenv to properly use the site-packages
on the host if a library is not available in the venv.

Change-Id: Ib84d48e8826bb96060338b3fa0782620c98794a8
Related-Bug: #1524684
Closes-Bug: #1529434

docker/aodh/aodh-base/Dockerfile.j2

@@ -15,7 +15,7 @@ RUN yum -y install \
 ADD aodh-base-archive /aodh-base-source
 RUN ln -s aodh-base-source/* aodh \
     && useradd --user-group aodh \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /aodh \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /aodh \
     && mkdir -p /etc/aodh /var/log/aodh /home/aodh \
     && cp -r /aodh/etc/* /etc/aodh/ \
     && chown -R aodh: /etc/aodh /var/log/aodh /home/aodh

docker/ceilometer/ceilometer-base/Dockerfile.j2

@@ -15,7 +15,7 @@ RUN yum -y install \
 ADD ceilometer-base-archive /ceilometer-base-source
 RUN ln -s ceilometer-base-source/* ceilometer \
     && useradd --user-group ceilometer \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /ceilometer \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /ceilometer \
     && mkdir -p /etc/ceilometer /var/log/ceilometer /home/ceilometer \
     && cp -r /ceilometer/etc/* /etc/ceilometer/ \
     && chown -R ceilometer: /etc/ceilometer /var/log/ceilometer /home/ceilometer

docker/cinder/cinder-base/Dockerfile.j2

@@ -31,7 +31,7 @@ RUN apt-get install -y --no-install-recommends \
 ADD cinder-base-archive /cinder-base-source
 RUN ln -s cinder-base-source/* cinder \
     && useradd --user-group cinder \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /cinder \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /cinder \
     && mkdir -p /etc/cinder /var/log/cinder /home/cinder \
     && cp -r /cinder/etc/cinder/* /etc/cinder/ \
     && chown -R cinder: /etc/cinder /var/log/cinder /home/cinder

docker/designate/designate-base/Dockerfile.j2

@@ -23,7 +23,7 @@ RUN yum install -y \
 ADD designate-base-archive /designate-base-source
 RUN ln -s designate-base-source/* designate \
     && useradd --user-group designate \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /designate \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /designate \
     && mkdir -p /etc/designate /var/log/designate /home/designate \
     && cp -r /designate/etc/designate/* /etc/designate/ \
     && chown -R designate: /etc/designate /var/log/designate /home/designate

docker/dind/Dockerfile.j2

@@ -18,7 +18,7 @@ RUN echo 'deb http://apt.dockerproject.org/repo ubuntu-trusty main' > /etc/apt/s
         docker-engine=1.8.2-0~trusty \
         openssh-server \
     && apt-get clean \
-    && pip --no-cache-dir install docker-py
+    && pip --no-cache-dir install --upgrade docker-py
 
 {% endif %}
 

docker/glance/glance-base/Dockerfile.j2

@@ -37,8 +37,7 @@ RUN apt-get install -y --no-install-recommends \
 ADD glance-base-archive /glance-base-source
 RUN ln -s glance-base-source/* glance \
     && useradd --user-group glance \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /glance \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install  python-cephlibs  /glance \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /glance \
     && mkdir -p /etc/glance /var/log/glance /var/lib/glance /home/glance \
     && cp -r /glance/etc/* /etc/glance/ \
     && chown -R glance: /etc/glance /var/log/glance /var/lib/glance /home/glance

docker/gnocchi/gnocchi-base/Dockerfile.j2

@@ -17,7 +17,7 @@ RUN apt-get install -y --no-install-recommends libpq-dev \
 ADD gnocchi-base-archive /gnocchi-base-source
 RUN ln -s gnocchi-base-source/* gnocchi \
     && useradd --user-group gnocchi \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /gnocchi \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /gnocchi \
     && mkdir -p /etc/gnocchi /var/log/gnocchi /home/gnocchi \
     && cp -r /gnocchi/etc/* /etc/gnocchi/ \
     && chown -R gnocchi: /etc/gnocchi /var/log/gnocchi /home/gnocchi

docker/heat/heat-base/Dockerfile.j2

@@ -14,7 +14,7 @@ RUN yum -y install \
 ADD heat-base-archive /heat-base-source
 RUN ln -s heat-base-source/* heat \
     && useradd --user-group heat \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /heat \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /heat \
     && mkdir -p /etc/heat /var/log/heat /home/heat \
     && cp -r /heat/etc/heat/* /etc/heat/ \
     && chown -R heat: /etc/heat /var/log/heat /home/heat

docker/horizon/Dockerfile.j2

@@ -43,7 +43,7 @@ RUN apt-get install -y --no-install-recommends \
 
 ADD horizon-archive /horizon-source
 RUN ln -s horizon-source/* horizon \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /horizon \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /horizon \
     && useradd --user-group horizon \
     && mkdir -p /etc/openstack-dashboard /home/horizon \
     && ln -s /etc/openstack-dashboard/local_settings /var/lib/kolla/venv/lib/python2.7/site-packages/openstack_dashboard/local/local_settings.py \

docker/ironic/ironic-base/Dockerfile.j2

@@ -20,7 +20,7 @@ RUN yum -y install \
 ADD ironic-base-archive /ironic-base-source
 RUN ln -s ironic-base-source/* ironic \
     && useradd --user-group ironic \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /ironic \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /ironic \
     && mkdir -p /etc/ironic /var/log/ironic /home/ironic \
     && cp -r /ironic/etc/ironic/* /etc/ironic/ \
     && chown -R ironic: /etc/ironic /var/log/ironic /home/ironic

docker/ironic/ironic-discoverd/Dockerfile.j2

@@ -4,7 +4,7 @@ MAINTAINER {{ maintainer }}
 {% if install_type == 'binary' %}
     {% if base_distro in ['centos', 'fedora', 'oraclelinux', 'rhel'] %}
 
-RUN /var/lib/kolla/venv/bin/pip install ironic-discoverd
+RUN pip install ironic-discoverd
 # discoverd no longer in delorean 9/28/2015, switch to inspector on TODO
 #RUN yum -y install \
 #    openstack-ironic-discoverd \

docker/keystone/Dockerfile.j2

@@ -38,7 +38,7 @@ RUN apt-get install -y --no-install-recommends \
 ADD keystone-archive /keystone-source
 RUN ln -s keystone-source/* keystone \
     && useradd --user-group keystone \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /keystone \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /keystone \
     && mkdir -p /etc/keystone /var/www/cgi-bin/keystone /var/log/apache2 /home/keystone \
     && cp -r /keystone/etc/* /etc/keystone/ \
     && cp /keystone/httpd/keystone.py /var/www/cgi-bin/keystone/admin \

docker/kolla-ansible/Dockerfile.j2

@@ -36,7 +36,7 @@ RUN apt-get -y --no-install-recommends install \
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
     && python get-pip.py \
     && rm get-pip.py \
-    && pip install -U wheel \
+    && pip --no-cache-dir install --upgrade wheel \
     && pip --no-cache-dir install \
         MySQL-python \
         os-client-config==1.11.1 \

docker/magnum/magnum-base/Dockerfile.j2

@@ -27,7 +27,7 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
 ADD magnum-base-archive /magnum-base-source
 RUN ln -s magnum-base-source/* magnum \
     && useradd --user-group magnum \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /magnum \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /magnum \
     && mkdir -p /etc/magnum /var/log/magnum /home/magnum \
     && cp -r /magnum/etc/magnum/* /etc/magnum \
     && chown -R magnum: /etc/magnum /var/log/magnum /home/magnum

docker/murano/murano-base/Dockerfile.j2

@@ -11,7 +11,7 @@ RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
 ADD murano-base-archive /murano-base-source
 RUN ln -s murano-base-source/* murano \
     && useradd --user-group murano \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /murano \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /murano \
     && mkdir -p /etc/murano /var/log/murano /home/murano \
     && cp -r /murano/etc/murano/* /etc/murano/ \
     && chown -R murano: /etc/murano /var/log/murano /home/murano

docker/neutron/neutron-base/Dockerfile.j2

@@ -39,7 +39,7 @@ RUN apt-get install -y --no-install-recommends \
 ADD neutron-base-archive /neutron-base-source
 RUN ln -s neutron-base-source/* neutron \
     && useradd --user-group neutron \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /neutron \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /neutron \
     && mkdir -p /etc/neutron /usr/share/neutron /var/log/neutron /home/neutron \
     && cp -r /neutron/etc/* /etc/neutron/ \
     && cp -r /neutron/etc/neutron/* /etc/neutron/ \

docker/neutron/neutron-server/Dockerfile.j2

@@ -5,7 +5,7 @@ ADD plugins-archive /
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 RUN chmod 755 /usr/local/bin/kolla_extend_start \
     && if [[ "$(ls /plugins)" ]]; then \
-           pip --no-cache-dir install -c requirements/upper-constraints.txt /plugins/*; \
+           pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /plugins/*; \
        fi
 
 {{ include_footer }}

docker/nova/nova-base/Dockerfile.j2

@@ -39,7 +39,7 @@ RUN apt-get install -y --no-install-recommends \
 ADD nova-base-archive /nova-base-source
 RUN ln -s nova-base-source/* nova \
     && useradd --user-group nova \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt /nova \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt /nova \
     && mkdir -p /etc/nova /var/log/nova /home/nova /var/lib/nova \
     && cp -r /nova/etc/nova/* /etc/nova/ \
     && chown -R nova: /etc/nova /var/log/nova /home/nova /var/lib/nova

docker/nova/nova-compute/Dockerfile.j2

@@ -34,7 +34,7 @@ RUN apt-get install -y --no-install-recommends \
 
     {% endif %}
 
-RUN /var/lib/kolla/venv/bin/pip --no-cache-dir install -c requirements/upper-constraints.txt libvirt-python
+RUN /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt libvirt-python
 
 {% endif %}
 

docker/openstack-base/Dockerfile.j2

@@ -79,9 +79,9 @@ RUN ln -s openstack-base-source/* /requirements \
     && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
     && python get-pip.py \
     && rm get-pip.py \
-    && pip install -U virtualenv \
-    && virtualenv /var/lib/kolla/venv \
-    && /var/lib/kolla/venv/bin/pip --no-cache-dir install -U -c requirements/upper-constraints.txt \
+    && pip --no-cache-dir install -U virtualenv \
+    && virtualenv --system-site-packages /var/lib/kolla/venv \
+    && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt \
         jinja2 \
         kazoo \
         python-barbicanclient \