kfatehi/README.md

## README.md

      
    Raw
  

              README.md
            
          
    Graphite + Carbon

An all-in-one image running graphite and carbon-cache. Version: 0.9.12.
This script contains a sensible default configuration of graphite and
carbon-cache. Starting this container will, by default, bind the the following
host ports:

80: the graphite web interface
2003: the carbon-cache line receiver (the standard graphite protocol)
2004: the carbon-cache pickle receiver
7002: the carbon-cache query port (used by the web interface)

You can log into the administrative interface of graphite-web (a Django
application) with the username admin and password admin. These passwords can
be changed through the web interface.
Data volumes

Graphite data is stored at /var/lib/graphite/storage/whisper
Technical details

By default, this instance of carbon-cache uses the following retention periods
resulting in whisper files of approximately 2.5MiB.
10s:8d,1m:31d,10m:1y,1h:5y

For the docker image, see the original repository.

  
## install-graphite-ubuntu-12.04.sh
# derived from https://github.com/nickstenning/dockerfiles/blob/master/graphite/Dockerfile

apt-get -y update
apt-get -y install python-ldap python-cairo python-django python-twisted python-django-tagging python-simplejson python-memcache python-pysqlite2 python-support python-pip gunicorn supervisor nginx-light

# Install required packages
pip install whisper
pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon
pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web

# Add system service config
cat << EOF > /etc/nginx/nginx.conf
user www-data;
worker_processes 1;
pid /var/run/nginx.pid;

events {
  worker_connections 1024;
}

http {
  sendfile on;
  tcp_nopush on;
  tcp_nodelay on;
  keepalive_timeout 65;
  types_hash_max_size 2048;
  server_tokens off;

  server_names_hash_bucket_size 32;

  include /etc/nginx/mime.types;
  default_type application/octet-stream;

  access_log /var/log/nginx/access.log;
  error_log /var/log/nginx/error.log;

  gzip on;
  gzip_disable "msie6";

  server {
    listen 80 default_server;
    server_name _;

    open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m;

    location / {
        proxy_pass                 http://127.0.0.1:8000;
        proxy_set_header           X-Real-IP   \$remote_addr;
        proxy_set_header           X-Forwarded-For  \$proxy_add_x_forwarded_for;
        proxy_set_header           X-Forwarded-Proto  \$scheme;
        proxy_set_header           X-Forwarded-Server  \$host;
        proxy_set_header           X-Forwarded-Host  \$host;
        proxy_set_header           Host  \$host;

        client_max_body_size       10m;
        client_body_buffer_size    128k;

        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;

        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }

    add_header Access-Control-Allow-Origin "*";
    add_header Access-Control-Allow-Methods "GET, OPTIONS";
    add_header Access-Control-Allow-Headers "origin, authorization, accept";

    location /content {
      alias /var/lib/graphite/webapp/content;
    }

    location /media {
      alias /usr/share/pyshared/django/contrib/admin/media;
    }
  }
}
EOF

cat << EOF > /etc/supervisor/conf.d/supervisord.conf
[supervisord]
environment = GRAPHITE_STORAGE_DIR='/var/lib/graphite/storage',GRAPHITE_CONF_DIR='/var/lib/graphite/conf'

[program:nginx]
command = /usr/sbin/nginx
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true

[program:carbon-cache]
user = www-data
command = /var/lib/graphite/bin/carbon-cache.py --debug start
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true

[program:graphite-webapp]
user = www-data
directory = /var/lib/graphite/webapp
environment = PYTHONPATH='/var/lib/graphite/webapp'
command = /usr/bin/gunicorn_django -b127.0.0.1:8000 -w2 graphite/settings.py
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true
EOF

# Add graphite config
cat << EOF > /var/lib/graphite/webapp/graphite/initial_data.json
[
  {
    "pk": 1,
    "model": "auth.user",
    "fields": {
      "username": "admin",
      "first_name": "",
      "last_name": "",
      "is_active": true,
      "is_superuser": true,
      "is_staff": true,
      "last_login": "2011-09-20 17:02:14",
      "groups": [],
      "user_permissions": [],
      "password": "sha1\$1b11b\$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236",
      "email": "[email protected]",
      "date_joined": "2011-09-20 17:02:14"
    }
  }
]
EOF

cat << EOF > /var/lib/graphite/webapp/graphite/local_settings.py
# Edit this file to override the default graphite settings, do not edit settings.py

# Turn on debugging and restart apache if you ever see an "Internal Server Error" page
#DEBUG = True

# Set your local timezone (django will try to figure this out automatically)
TIME_ZONE = 'UTC'

# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely
#MEMCACHE_HOSTS = ['127.0.0.1:11211']

# Sometimes you need to do a lot of rendering work but cannot share your storage mount
#REMOTE_RENDERING = True
#RENDERING_HOSTS = ['fastserver01','fastserver02']
#LOG_RENDERING_PERFORMANCE = True
#LOG_CACHE_PERFORMANCE = True

# If you've got more than one backend server they should all be listed here
#CLUSTER_SERVERS = []

# Override this if you need to provide documentation specific to your graphite deployment
#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite"

# Enable email-related features
#SMTP_SERVER = "mail.mycompany.com"

# LDAP / ActiveDirectory authentication setup
#USE_LDAP_AUTH = True
#LDAP_SERVER = "ldap.mycompany.com"
#LDAP_PORT = 389
#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com"
#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com"
#LDAP_BASE_PASS = "readonly_account_password"
#LDAP_USER_QUERY = "(username=%s)"  #For Active Directory use "(sAMAccountName=%s)"

# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!)
#DATABASE_ENGINE = 'mysql' # or 'postgres'
#DATABASE_NAME = 'graphite'
#DATABASE_USER = 'graphite'
#DATABASE_PASSWORD = 'graphite-is-awesome'
#DATABASE_HOST = 'mysql.mycompany.com'
#DATABASE_PORT = '3306'
EOF

cat << EOF > /var/lib/graphite/conf/carbon.conf
[cache]
LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/

# Specify the user to drop privileges to
# If this is blank carbon runs as the user that invokes it
# This user must have write access to the local data directory
USER =

# Limit the size of the cache to avoid swapping or becoming CPU bound.
# Sorts and serving cache queries gets more expensive as the cache grows.
# Use the value "inf" (infinity) for an unlimited cache size.
MAX_CACHE_SIZE = inf

# Limits the number of whisper update_many() calls per second, which effectively
# means the number of write requests sent to the disk. This is intended to
# prevent over-utilizing the disk and thus starving the rest of the system.
# When the rate of required updates exceeds this, then carbon's caching will
# take effect and increase the overall throughput accordingly.
MAX_UPDATES_PER_SECOND = 1000

# Softly limits the number of whisper files that get created each minute.
# Setting this value low (like at 50) is a good way to ensure your graphite
# system will not be adversely impacted when a bunch of new metrics are
# sent to it. The trade off is that it will take much longer for those metrics'
# database files to all get created and thus longer until the data becomes usable.
# Setting this value high (like "inf" for infinity) will cause graphite to create
# the files quickly but at the risk of slowing I/O down considerably for a while.
MAX_CREATES_PER_MINUTE = inf

LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2003

PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2004

CACHE_QUERY_INTERFACE = 0.0.0.0
CACHE_QUERY_PORT = 7002

LOG_UPDATES = False

# Enable AMQP if you want to receve metrics using an amqp broker
# ENABLE_AMQP = False

# Verbose means a line will be logged for every metric received
# useful for testing
# AMQP_VERBOSE = False

# AMQP_HOST = localhost
# AMQP_PORT = 5672
# AMQP_VHOST = /
# AMQP_USER = guest
# AMQP_PASSWORD = guest
# AMQP_EXCHANGE = graphite

# Patterns for all of the metrics this machine will store. Read more at
# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
#
# Example: store all sales, linux servers, and utilization metrics
# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
#
# Example: store everything
# BIND_PATTERNS = #

# NOTE: you cannot run both a cache and a relay on the same server
# with the default configuration, you have to specify a distinict
# interfaces and ports for the listeners.

[relay]
LINE_RECEIVER_INTERFACE = 0.0.0.0
LINE_RECEIVER_PORT = 2003

PICKLE_RECEIVER_INTERFACE = 0.0.0.0
PICKLE_RECEIVER_PORT = 2004

CACHE_SERVERS = server1, server2, server3
MAX_QUEUE_SIZE = 10000
EOF

cat << EOF > /var/lib/graphite/conf/storage-schemas.conf
[carbon]
pattern = ^carbon\..*
retentions = 1m:31d,10m:1y,1h:5y

[default]
pattern = .*
retentions = 10s:8d,1m:31d,10m:1y,1h:5y
EOF

mkdir -p /var/lib/graphite/storage/whisper
touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index
chown -R www-data /var/lib/graphite/storage
chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper
chmod 0664 /var/lib/graphite/storage/graphite.db
cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput

# Nginx
ufw allow 80
# Carbon line receiver port
ufw allow 2003
# Carbon pickle receiver port
ufw allow 2004
# Carbon cache query port
ufw allow 7002

service supervisor stop
service supervisor start
	# derived from https://github.com/nickstenning/dockerfiles/blob/master/graphite/Dockerfile

	apt-get -y update
	apt-get -y install python-ldap python-cairo python-django python-twisted python-django-tagging python-simplejson python-memcache python-pysqlite2 python-support python-pip gunicorn supervisor nginx-light

	# Install required packages
	pip install whisper
	pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon
	pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web

	# Add system service config
	cat << EOF > /etc/nginx/nginx.conf
	user www-data;
	worker_processes 1;
	pid /var/run/nginx.pid;

	events {
	worker_connections 1024;
	}

	http {
	sendfile on;
	tcp_nopush on;
	tcp_nodelay on;
	keepalive_timeout 65;
	types_hash_max_size 2048;
	server_tokens off;

	server_names_hash_bucket_size 32;

	include /etc/nginx/mime.types;
	default_type application/octet-stream;

	access_log /var/log/nginx/access.log;
	error_log /var/log/nginx/error.log;

	gzip on;
	gzip_disable "msie6";

	server {
	listen 80 default_server;
	server_name _;

	open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m;

	location / {
	proxy_pass http://127.0.0.1:8000;
	proxy_set_header X-Real-IP \$remote_addr;
	proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
	proxy_set_header X-Forwarded-Proto \$scheme;
	proxy_set_header X-Forwarded-Server \$host;
	proxy_set_header X-Forwarded-Host \$host;
	proxy_set_header Host \$host;

	client_max_body_size 10m;
	client_body_buffer_size 128k;

	proxy_connect_timeout 90;
	proxy_send_timeout 90;
	proxy_read_timeout 90;

	proxy_buffer_size 4k;
	proxy_buffers 4 32k;
	proxy_busy_buffers_size 64k;
	proxy_temp_file_write_size 64k;
	}

	add_header Access-Control-Allow-Origin "*";
	add_header Access-Control-Allow-Methods "GET, OPTIONS";
	add_header Access-Control-Allow-Headers "origin, authorization, accept";

	location /content {
	alias /var/lib/graphite/webapp/content;
	}

	location /media {
	alias /usr/share/pyshared/django/contrib/admin/media;
	}
	}
	}
	EOF

	cat << EOF > /etc/supervisor/conf.d/supervisord.conf
	[supervisord]
	environment = GRAPHITE_STORAGE_DIR='/var/lib/graphite/storage',GRAPHITE_CONF_DIR='/var/lib/graphite/conf'

	[program:nginx]
	command = /usr/sbin/nginx
	stdout_logfile = /var/log/supervisor/%(program_name)s.log
	stderr_logfile = /var/log/supervisor/%(program_name)s.log
	autorestart = true

	[program:carbon-cache]
	user = www-data
	command = /var/lib/graphite/bin/carbon-cache.py --debug start
	stdout_logfile = /var/log/supervisor/%(program_name)s.log
	stderr_logfile = /var/log/supervisor/%(program_name)s.log
	autorestart = true

	[program:graphite-webapp]
	user = www-data
	directory = /var/lib/graphite/webapp
	environment = PYTHONPATH='/var/lib/graphite/webapp'
	command = /usr/bin/gunicorn_django -b127.0.0.1:8000 -w2 graphite/settings.py
	stdout_logfile = /var/log/supervisor/%(program_name)s.log
	stderr_logfile = /var/log/supervisor/%(program_name)s.log
	autorestart = true
	EOF

	# Add graphite config
	cat << EOF > /var/lib/graphite/webapp/graphite/initial_data.json
	[
	{
	"pk": 1,
	"model": "auth.user",
	"fields": {
	"username": "admin",
	"first_name": "",
	"last_name": "",
	"is_active": true,
	"is_superuser": true,
	"is_staff": true,
	"last_login": "2011-09-20 17:02:14",
	"groups": [],
	"user_permissions": [],
	"password": "sha1\$1b11b\$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236",
	"email": "[email protected]",
	"date_joined": "2011-09-20 17:02:14"
	}
	}
	]
	EOF

	cat << EOF > /var/lib/graphite/webapp/graphite/local_settings.py
	# Edit this file to override the default graphite settings, do not edit settings.py

	# Turn on debugging and restart apache if you ever see an "Internal Server Error" page
	#DEBUG = True

	# Set your local timezone (django will try to figure this out automatically)
	TIME_ZONE = 'UTC'

	# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely
	#MEMCACHE_HOSTS = ['127.0.0.1:11211']

	# Sometimes you need to do a lot of rendering work but cannot share your storage mount
	#REMOTE_RENDERING = True
	#RENDERING_HOSTS = ['fastserver01','fastserver02']
	#LOG_RENDERING_PERFORMANCE = True
	#LOG_CACHE_PERFORMANCE = True

	# If you've got more than one backend server they should all be listed here
	#CLUSTER_SERVERS = []

	# Override this if you need to provide documentation specific to your graphite deployment
	#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite"

	# Enable email-related features
	#SMTP_SERVER = "mail.mycompany.com"

	# LDAP / ActiveDirectory authentication setup
	#USE_LDAP_AUTH = True
	#LDAP_SERVER = "ldap.mycompany.com"
	#LDAP_PORT = 389
	#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com"
	#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com"
	#LDAP_BASE_PASS = "readonly_account_password"
	#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)"

	# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!)
	#DATABASE_ENGINE = 'mysql' # or 'postgres'
	#DATABASE_NAME = 'graphite'
	#DATABASE_USER = 'graphite'
	#DATABASE_PASSWORD = 'graphite-is-awesome'
	#DATABASE_HOST = 'mysql.mycompany.com'
	#DATABASE_PORT = '3306'
	EOF

	cat << EOF > /var/lib/graphite/conf/carbon.conf
	[cache]
	LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/

	# Specify the user to drop privileges to
	# If this is blank carbon runs as the user that invokes it
	# This user must have write access to the local data directory
	USER =

	# Limit the size of the cache to avoid swapping or becoming CPU bound.
	# Sorts and serving cache queries gets more expensive as the cache grows.
	# Use the value "inf" (infinity) for an unlimited cache size.
	MAX_CACHE_SIZE = inf

	# Limits the number of whisper update_many() calls per second, which effectively
	# means the number of write requests sent to the disk. This is intended to
	# prevent over-utilizing the disk and thus starving the rest of the system.
	# When the rate of required updates exceeds this, then carbon's caching will
	# take effect and increase the overall throughput accordingly.
	MAX_UPDATES_PER_SECOND = 1000

	# Softly limits the number of whisper files that get created each minute.
	# Setting this value low (like at 50) is a good way to ensure your graphite
	# system will not be adversely impacted when a bunch of new metrics are
	# sent to it. The trade off is that it will take much longer for those metrics'
	# database files to all get created and thus longer until the data becomes usable.
	# Setting this value high (like "inf" for infinity) will cause graphite to create
	# the files quickly but at the risk of slowing I/O down considerably for a while.
	MAX_CREATES_PER_MINUTE = inf

	LINE_RECEIVER_INTERFACE = 0.0.0.0
	LINE_RECEIVER_PORT = 2003

	PICKLE_RECEIVER_INTERFACE = 0.0.0.0
	PICKLE_RECEIVER_PORT = 2004

	CACHE_QUERY_INTERFACE = 0.0.0.0
	CACHE_QUERY_PORT = 7002

	LOG_UPDATES = False

	# Enable AMQP if you want to receve metrics using an amqp broker
	# ENABLE_AMQP = False

	# Verbose means a line will be logged for every metric received
	# useful for testing
	# AMQP_VERBOSE = False

	# AMQP_HOST = localhost
	# AMQP_PORT = 5672
	# AMQP_VHOST = /
	# AMQP_USER = guest
	# AMQP_PASSWORD = guest
	# AMQP_EXCHANGE = graphite

	# Patterns for all of the metrics this machine will store. Read more at
	# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
	#
	# Example: store all sales, linux servers, and utilization metrics
	# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
	#
	# Example: store everything
	# BIND_PATTERNS = #

	# NOTE: you cannot run both a cache and a relay on the same server
	# with the default configuration, you have to specify a distinict
	# interfaces and ports for the listeners.

	[relay]
	LINE_RECEIVER_INTERFACE = 0.0.0.0
	LINE_RECEIVER_PORT = 2003

	PICKLE_RECEIVER_INTERFACE = 0.0.0.0
	PICKLE_RECEIVER_PORT = 2004

	CACHE_SERVERS = server1, server2, server3
	MAX_QUEUE_SIZE = 10000
	EOF

	cat << EOF > /var/lib/graphite/conf/storage-schemas.conf
	[carbon]
	pattern = ^carbon\..*
	retentions = 1m:31d,10m:1y,1h:5y

	[default]
	pattern = .*
	retentions = 10s:8d,1m:31d,10m:1y,1h:5y
	EOF

	mkdir -p /var/lib/graphite/storage/whisper
	touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index
	chown -R www-data /var/lib/graphite/storage
	chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper
	chmod 0664 /var/lib/graphite/storage/graphite.db
	cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput

	# Nginx
	ufw allow 80
	# Carbon line receiver port
	ufw allow 2003
	# Carbon pickle receiver port
	ufw allow 2004
	# Carbon cache query port
	ufw allow 7002

	service supervisor stop
	service supervisor start