Deepdream installation

deepdream-install.md

#!/usr/bin/env bash

# Assuming OS X Yosemite 10.10.4

# Install XCode and command line tools
# See https://itunes.apple.com/us/app/xcode/id497799835?mt=12#
# See https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/xcode-select.1.html
xcode-select --install

# Install the Homebrew package manager if you don't already use it; see source http://brew.sh
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"

# Install homebrew cask
brew install caskroom/cask/brew-cask

# Install Python 2.7 and pip
brew install python pip

# Install Cuda 7
brew cask install cuda

# Optionally install cuDNN from https://developer.nvidia.com/cudnn which requries
# 1) Registering for an NVIDIA account, accepting an end-user-license-agreement
# 2) Downloading and extracting a tar.gz archive
# 3) Copying lib* files to /usr/local/cuda/lib and cudnn.h to /usr/local/cuda/include
#
# robertsdionne:~/$ ls /usr/local/cuda/lib/*dnn*
# /usr/local/cuda/lib/libcudnn.6.5.dylib /usr/local/cuda/lib/libcudnn_static.a
# /usr/local/cuda/lib/libcudnn.dylib
# robertsdionne:~/$ ls /usr/local/cuda/include/*dnn*
# /usr/local/cuda/include/cudnn.h

# Install dependencies; see http://caffe.berkeleyvision.org/install_osx.html
brew install --fresh -vd snappy leveldb gflags glog szip lmdb
# need the homebrew science source for OpenCV and hdf5
brew tap homebrew/science
brew install hdf5 opencv
# with Python pycaffe needs dependencies built from source
brew install --build-from-source --with-python --fresh -vd protobuf
# We need boost{,-python}-1.57.0 due to a bug in 1.58.0 on Yosemite 10.10.4
brew install --build-from-source --fresh -vd \
    https://raw.githubusercontent.com/Homebrew/homebrew/6fd6a9b6b2f56139a44dd689d30b7168ac13effb/Library/Formula/boost.rb \
    https://raw.githubusercontent.com/Homebrew/homebrew/3141234b3473717e87f3958d4916fe0ada0baba9/Library/Formula/boost-python.rb

# Clone caffe
git clone https://github.com/BVLC/caffe.git
cd caffe

# Install python dependencies
pip install --requirement python/requirements.txt

# Adjust Makefile.config
cp Makefile.config.example Makefile.config

########## Make the following adjustments to Makefile.config:

diff --git a/Makefile.config b/../caffe/Makefile.config
index a873502..c6806c2 100644
--- a/Makefile.config
+++ b/../caffe/Makefile.config
@@ -2,7 +2,7 @@
 # Contributions simplifying and improving our build system are welcome!
 
 # cuDNN acceleration switch (uncomment to build with cuDNN).
-# USE_CUDNN := 1
+# USE_CUDNN := 1  ########### Keep it like this to disable cuDNN
+USE_CUDNN := 1    ########### Make it like this to enable cuDNN
 
 # CPU-only switch (uncomment to build without GPU support).
 # CPU_ONLY := 1
@@ -34,7 +34,7 @@ BLAS := atlas
 # Custom (MKL/ATLAS/OpenBLAS) include and lib directories.
 # Leave commented to accept the defaults for your choice of BLAS
 # (which should work)!
-# BLAS_INCLUDE := /path/to/your/blas
+BLAS_INCLUDE := /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.10.sdk/System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Headers
 # BLAS_LIB := /path/to/your/blas
 
 # Homebrew puts openblas in a directory that is not on the standard search path
@@ -48,8 +48,8 @@ BLAS := atlas
 
 # NOTE: this is required only if you will compile the python interface.
 # We need to be able to find Python.h and numpy/arrayobject.h.
-PYTHON_INCLUDE := /usr/include/python2.7 \
-		/usr/lib/python2.7/dist-packages/numpy/core/include
+PYTHON_INCLUDE := /usr/local/lib/python2.7/site-packages/numpy/core/include/ \
+		/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/include/python2.7/
 # Anaconda Python distribution is quite popular. Include path:
 # Verify anaconda location, sometimes it's in root.
 # ANACONDA_HOME := $(HOME)/anaconda
@@ -58,7 +58,9 @@ PYTHON_INCLUDE := /usr/include/python2.7 \
 		# $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include \
 
 # We need to be able to find libpythonX.X.so or .dylib.
-PYTHON_LIB := /usr/lib
+PYTHON_LIB := /usr/local/lib/python2.7 \
+		/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/lib/ \
+		/usr/local/lib/python2.7/site-packages/numpy/lib
 # PYTHON_LIB := $(ANACONDA_HOME)/lib
 
 # Homebrew installs numpy in a non standard path (keg only)

########## Make the following adjustments to Makefile:

diff --git a/Makefile b/../caffe/Makefile
index e4e66df..8babaf8 100644
--- a/Makefile
+++ b/../caffe/Makefile
@@ -335,7 +335,7 @@ else
 			LDFLAGS += -framework Accelerate
 		else
 			BLAS_INCLUDE ?= /System/Library/Frameworks/vecLib.framework/Versions/Current/Headers/
-			LDFLAGS += -framework vecLib
+			LDFLAGS += -framework Accelerate
 		endif
 	endif
 endif

# Build caffe
make all -j8; make pycaffe -j8

# Package caffe
make distribute

# Set your environment variables for success
export PYTHONPATH="$HOME/caffe/distribute/python:$PYTHONPATH"
export DYLD_FALLBACK_LIBRARY_PATH=/usr/local/cuda/lib:/usr/local/lib:$HOME/caffe/distribute/lib:$DYLD_FALLBACK_LIBRARY_PATH

# Clone deepdream
cd ..
git clone https://github.com/google/deepdream.git
cd deepdream

# Install python requirements
pip install numpy scipy scikit-image pillow ipython[notebook]

# Start IPython Notebook and select dream.ipynb
ipython notebook dream.ipynb

@robertsdionne thanks a bunch for the help on my previous question. Having another issue unfortunately.

linking warning saying that xcode can't find the Python library leading to the build failing because it cant find Python.h

make all -j8; make pycaffe -j8
...
CXX examples/siamese/convert_mnist_siamese_data.cpp
CXX .build_release/src/caffe/proto/caffe.pb.cc
AR -o .build_release/lib/libcaffe.a
LD -o .build_release/lib/libcaffe.so
clang: warning: argument unused during compilation: '-pthread'
ld: warning: directory not found for option '-L/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/lib/'
CXX/LD -o .build_release/tools/caffe.bin
.... plus lots more ld warnings
PROTOC (python) src/caffe/proto/caffe.proto
python/caffe/_caffe.cpp:1:10: fatal error: 'Python.h' file not found

include <Python.h> // NOLINT(build/include_alpha)

The directory is there: $ ls /usr/local/Cellar/python/2.7.10_1/Frameworks/Python.framework/Versions/2.7/lib/
libpython2.7.dylib pkgconfig/ python2.7/

Makefile:
...
PYTHON_INCLUDE := /usr/local/lib/python2.7/site-packages/numpy/core/include/
/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/include/python2.7/
...
PYTHON_LIB := /usr/local/lib/python2.7
/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/lib/
/usr/local/lib/python2.7/site-packages/numpy/lib
...

Still having issues. When I use "make all -j8; make pycaffe -j8" I get the errors listed via the link. Can anyone please help?
http://pastebin.com/LHX2xsDj

When I run the notebook, or even just start the Python interpreter and type "import caffe" I get a segfault:

Exception Type: EXC_BAD_ACCESS (SIGSEGV)
Exception Codes: KERN_INVALID_ADDRESS at 0x0000000000000000

VM Regions Near 0:
-->
__TEXT 0000000102ac1000-0000000102ac3000 [ 8K] r-x/rwx SM=COW /usr/local/Cellar/python/2.7.9/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python

This is Python 2.7.9.
Any ideas?

just a tip for those having problems, I was able to resolve my problems by updating XCode to the latest version, 6.4.

Make sure you update if you are running 6.3 or earlier!

was able to open the notebook, but when i try "run" on it, i appear to have the following error:

7 from google.protobuf import text_format
8
----> 9 import caffe
10
11 def showarray(a, fmt='jpeg'):
ImportError: No module named caffe

Could somebody help me, please?

Hi fellow #deepdream ers
Got pretty far in this, and ran into some sags at the end.
When I get to:
make all -j8; make pycaffe -j8
(in my case: make all -j4; make pycaffe -j4 because I have 4 core and not 8) I am getting error

./include/caffe/util/cudnn.hpp:5:10: fatal error: 'cudnn.h' file not found

include < cudnn.h >

and can't seem to get past it. Tried it on my other computer too, same thing.
Any suggestions?
I see some other people had a similar issue but I can't seem to find an answer? Did any of you find a resolution or work-around?
I have Xcode 6.4 btw

Hey @aidanfowler, were you able to figure out the python.h issue? I am seeing that problem.

I am having the same problem with the python.h. among others. I noticed on my machine that frameworks is in folder 2.7.10_1
From what aidanfowler posted:
The directory is there: $ ls /usr/local/Cellar/python/2.7.10_1/Frameworks/Python.framework/Versions/2.7/lib/
libpython2.7.dylib pkgconfig/ python2.7/

check your file path for 2.7.10 or 2.7.10_1
Should the makefile have 2.7.10 replaced with 2.7.10_1 like as follows?
...
PYTHON_INCLUDE := /usr/local/lib/python2.7/site-packages/numpy/core/include/
/usr/local/Cellar/python/2.7.10_1/Frameworks/Python.framework/Versions/2.7/include/python2.7/
...
PYTHON_LIB := /usr/local/lib/python2.7
/usr/local/Cellar/python/2.7.10_1/Frameworks/Python.framework/Versions/2.7/lib/
/usr/local/lib/python2.7/site-packages/numpy/lib

I'm getting this error:
Traceback (most recent call last):
File "/usr/local/bin/ipython", line 7, in
from IPython import start_ipython
File "/usr/local/lib/python2.7/site-packages/IPython/init.py", line 45, in
from .config.loader import Config
File "/usr/local/lib/python2.7/site-packages/IPython/config/init.py", line 6, in
from .application import *
File "/usr/local/lib/python2.7/site-packages/IPython/config/application.py", line 19, in
from IPython.config.configurable import SingletonConfigurable
File "/usr/local/lib/python2.7/site-packages/IPython/config/configurable.py", line 12, in
from .loader import Config, LazyConfigValue
File "/usr/local/lib/python2.7/site-packages/IPython/config/loader.py", line 16, in
from IPython.utils.path import filefind, get_ipython_dir
File "/usr/local/lib/python2.7/site-packages/IPython/utils/path.py", line 14, in
import tempfile
File "/usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/lib/python2.7/tempfile.py", line 32, in
import io as _io
File "/Users/cedac/caffe/python/caffe/io.py", line 1, in
import numpy as np
File "/usr/local/lib/python2.7/site-packages/numpy/init.py", line 170, in
from . import add_newdocs
File "/usr/local/lib/python2.7/site-packages/numpy/add_newdocs.py", line 13, in
from numpy.lib import add_newdoc
File "/usr/local/lib/python2.7/site-packages/numpy/lib/init.py", line 8, in
from .type_check import *
File "/usr/local/lib/python2.7/site-packages/numpy/lib/type_check.py", line 11, in
import numpy.core.numeric as _nx
File "/usr/local/lib/python2.7/site-packages/numpy/core/init.py", line 46, in
from numpy.testing import Tester
File "/usr/local/lib/python2.7/site-packages/numpy/testing/init.py", line 13, in
from .utils import *
File "/usr/local/lib/python2.7/site-packages/numpy/testing/utils.py", line 15, in
from tempfile import mkdtemp
ImportError: cannot import name mkdtemp

I think that the problem is that it is loading caffe io.py instead of system's one, does anyone know how to solve this issue?

Thanks! This worked, with some tweaks. @snwman, my python was also installed to 2.7.10_1, so I made those changes to my Makefile.config. Had to clean but seemed ok afterwards. Also had to update my environment variables to point to the correct caffe dir.

One other note, I had to download the googlenet caffe models:
scripts/download_model_binary.py models/bvlc_googlenet

Ok, after much headache, I got this set up and running. I've spent yesterday and today tinkering, and have been generating cool, creepy images.

But... I've been running this in Anaconda/Spyder. I somehow got a message in the IPython console that the connection to the kernel had crashed. I restarted Spyder, created a new IPython console, loaded the script (completely unchaged), and tried to run it. Now I am getting
ImportError: No module named caffe

It literally was just running, and had no problem finding the caffe module.
I thought maybe I'd corrupted the caffee install somehow, so reran 'make pycaffe', which told me
make: Nothing to be done for 'pycaffe'.

Any idea why it would suddenly lose it's memory of where to import caffe from, and how to make it remember?

i needed to install libpng1.6.17

==> Verifying boost-1.57.0.tar.bz2 checksum
Error: SHA1 mismatch
Expected: e151557ae47afd1b43dc3fac46f8b04a8fe51c12
Actual: ea60df615387f47bdfdfe4da48f6567c4ec60f74
Archive: /Library/Caches/Homebrew/boost-1.57.0.tar.bz2

I have edited /usr/local/Library/Formula/boost.rb to have the correct strings, but it still will not install, please help

class Boost < Formula
desc "Collection of portable C++ source libraries"
homepage "http://www.boost.org"
url "https://downloads.sourceforge.net/project/boost/boost/1.57.0/boost_1_57_0.tar.bz2"
sha1 "e151557ae47afd1b43dc3fac46f8b04a8fe51c12"
head "https://github.com/boostorg/boost.git"

bottle do
cellar :any
sha256 "9a3929bec0e9e9db36e005f57193433ac6b5ff9ff86b2ed3262b975d58488c19" => :yosemite
sha256 "c5ad1f8591ed91c551658fd198ce31cf9f6a8026fcbfd5970a39cb479e64faa2" => :mavericks
sha256 "7b154bf42d72bbb90c3017bb94b24fdd2e0605ceab8717283f5c9f456ac5c03d" => :mountain_lion
end

When i try running the make all j8; pycaffe j8 I get this error can someone please help.

http://pastebin.com/H8qh4kcL

I think it's better to use /usr/local/Frameworks/Python.framework/Versions/Current/include/python2.7 instead of /usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/include/python2.7/ in PYTHON_INCLUDE and similarily /usr/local/Frameworks/Python.framework/Versions/Current/lib instead of /usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/lib/ in PYTHON_LIB.

At least this works for me in Mac OS 10.10.3.

Having the same issue as @Levixodus on make all -j8; make pycaffe -j8. Error is here: https://gist.github.com/shayanjm/bd081eebff7371110ced

XCode v.6.4, Mac OS 10.10.4. Also tried using cmake which amounted to the same errors.

@Levixodus: Check which version of boost is being used. It might be that you have 1.57.0 installed underneath 1.58.0. You would have to brew uninstall boost && brew link boost assuming 1.57.0 and 1.58.0 are both installed & managed by homebrew.

I've seen quite a few with the google.protobuf error and not really seeing any solutions here. Has anyone been able to address this error? I have everything 'working', or so it seems. When i run ipython in my terminal and import caffe, it seems to work. When I make runtest caffe it all comes back OK. I've followed all of the steps and the only thing I can think is that it's possible my ENV's aren't set correctly. But I'm immediately stopped in my tracks after the notebook launches and in the first module there is the dreaded "ImportError: No module named google.protobuf" error and I'm stopped right there. Or, at least it seems that way. Im in Ubuntu, but at this stage I'm not sure that matters as much. Help?

brew install opencv didn't work for me (already had an opencv formula for whatever reason), brew install homebrew/science/opencv seems like the safer command

I'm having the same issue as @Teepareep:

import caffe
Traceback (most recent call last):
File "", line 1, in
File "/Users/wilco/git/caffe/distribute/python/caffe/init.py", line 1, in
from .pycaffe import Net, SGDSolver
File "/Users/wilco/git/caffe/distribute/python/caffe/pycaffe.py", line 13, in
from ._caffe import Net, SGDSolver
ImportError: dlopen(/Users/wilco/git/caffe/distribute/python/caffe/_caffe.so, 2): Library not loaded: @rpath/libcaffe.so
Referenced from: /Users/wilco/git/caffe/distribute/python/caffe/_caffe.so
Reason: image not found

I didn't see any errors during compiling...

For the Python.h

I replaced

+PYTHON_INCLUDE := /usr/local/lib/python2.7/site-packages/numpy/core/include/ \

•   /usr/local/Cellar/python/2.7.10/Frameworks/Python.framework/Versions/2.7/include/python2.7/
  

With

+PYTHON_INCLUDE := /usr/local/lib/python2.7/site-packages/numpy/core/include/ \

•  /usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/include/python2.7/
  

(just 2.7.10_2 instead of 2.7.10) don't know why I have this confusing path...

I keep getting this error when trying to "Make" Caffe

"$ make all -j8; make pycaffe -j8
Makefile.config:3: *** missing separator. Stop.
Makefile.config:3: *** missing separator. Stop."

I followed the directions closely... and everything up to this point worked.

(edit) I took the Makefile and Makefile.config and reverted them to the "factory" settings, and the install of Caffe looks to be working. whether that will work with deepdream, i don't know..

Awesome work to make this installation simpler!
I've followed it carefully.
When running $ make runtest or import caffe I'm getting a similar error to @Teepareep:

ImportError: dlopen(/Users/guigou/caffe/distribute/python/caffe/_caffe.so, 2): Library not loaded: @rpath/libcaffe.so
  Referenced from: /Users/guigou/caffe/distribute/python/caffe/_caffe.so
  Reason: image not found

any clue? Google seems to indicate to look at the DYLD_LIBRARY_PATH/DYLD_FALLBACK_LIBRARY_PATH but I could not change them successfully. DYLD_FALLBACK_LIBRARY_PATH is set as suggested in this post.

I'm running OSX 10.11 with anaconda and in the Makefile.config I've uncommented the lines

ANACONDA_HOME := $(HOME)/anaconda
PYTHON_INCLUDE := $(ANACONDA_HOME)/include \
    $(ANACONDA_HOME)/include/python2.7 \
    $(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include \

and set

PYTHON_LIB := $(ANACONDA_HOME)/lib

Don't know if that's relevant :)
Many thanks!

Thanks for this guide. I've followed the instructions and everything seemed to go smoothly. All the Caffe tests ran without issue. However when I try to do import caffe in python it crashes with:

Python 2.7.11 |Anaconda 2.5.0 (x86_64)| (default, Dec  6 2015, 18:57:58) 
[GCC 4.2.1 (Apple Inc. build 5577)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
Anaconda is brought to you by Continuum Analytics.
Please check out: http://continuum.io/thanks and https://anaconda.org
>>> import caffe
Fatal Python error: PyThreadState_Get: no current thread
Abort trap: 6

I'm using CUDnn and I ran make with -j4 option since my MBP has 4 cores. I'm guessing something is breaking down on the python side. Any ideas of what I should check? Is there a way to get more information out of the crash?

Hey I was following the steps exactly as you told however at the point

diff --git a/Makefile.config b/../caffe/Makefile.config

its showing me an error that "unrecognized option diff --get". I would be really grateful if someone could help me out as to what I should be doing over here. Many thanks.

After having almost all errors listed in this conversation, I had this issue I never seen and I don't know how to solve it, when I run the first cell of the ipython notebook

`---------------------------------------------------------------------------

ImportError Traceback (most recent call last)
in ()
7 from google.protobuf import text_format
8
----> 9 import caffe
10
11 # If your GPU supports CUDA and Caffe was built with CUDA support,

/Users/MNv/caffe/distribute/python/caffe/init.py in ()
----> 1 from .pycaffe import Net, SGDSolver, NesterovSolver, AdaGradSolver, RMSPropSolver, AdaDeltaSolver, AdamSolver
2 from ._caffe import set_mode_cpu, set_mode_gpu, set_device, Layer, get_solver, layer_type_list
3 from ._caffe import version
4 from .proto.caffe_pb2 import TRAIN, TEST
5 from .classifier import Classifier

/Users/MNv/caffe/distribute/python/caffe/pycaffe.py in ()
13 from ._caffe import Net, SGDSolver, NesterovSolver, AdaGradSolver,
14 RMSPropSolver, AdaDeltaSolver, AdamSolver
---> 15 import caffe.io
16
17 import six

/Users/MNv/caffe/distribute/python/caffe/io.py in ()
2 import skimage.io
3 from scipy.ndimage import zoom
----> 4 from skimage.transform import resize
5
6 try:

/usr/local/lib/python2.7/site-packages/skimage/transform/init.py in ()
----> 1 from .hough_transform import (hough_line, hough_line_peaks,
2 probabilistic_hough_line, hough_circle,
3 hough_ellipse)
4 from .radon_transform import radon, iradon, iradon_sart
5 from .finite_radon_transform import frt2, ifrt2

/usr/local/lib/python2.7/site-packages/skimage/transform/hough_transform.py in ()
2 from scipy import ndimage as ndi
3 from .. import measure
----> 4 from ._hough_transform import (_hough_circle,
5 hough_ellipse as _hough_ellipse,
6 hough_line as _hough_line,

skimage/transform/_hough_transform.pyx in init skimage.transform._hough_transform (skimage/transform/_hough_transform.c:23691)()

/usr/local/lib/python2.7/site-packages/skimage/draw/init.py in ()
----> 1 from .draw import circle, ellipse, polygon_perimeter, set_color
2 from .draw3d import ellipsoid, ellipsoid_stats
3 from ._draw import (line, line_aa, polygon, ellipse_perimeter,
4 circle_perimeter, circle_perimeter_aa,
5 _bezier_segment, bezier_curve)

/usr/local/lib/python2.7/site-packages/skimage/draw/draw.py in ()
4 from ._draw import _coords_inside_image
5
----> 6 from .._shared._geometry import polygon_clip
7 from ._draw import line
8

/usr/local/lib/python2.7/site-packages/skimage/_shared/_geometry.py in ()
2
3 import numpy as np
----> 4 from matplotlib import _path, path, transforms
5
6

/usr/local/lib/python2.7/site-packages/matplotlib/path.py in ()
24 from numpy import ma
25
---> 26 from matplotlib import _path
27 from matplotlib.cbook import simple_linear_interpolation, maxdict
28 from matplotlib import rcParams

ImportError: cannot import name _path`

I got the following issue when I run the command on Mac OS Sierra:

python -c "import caffe" 

and here is the error message

python(38945,0x7fffaf2063c0) malloc: *** malloc_zone_unregister() failed for 0x7fffaf1fc000
Segmentation fault: 11

I will appreciate a kind suggestion.