rauhs/ansible-core.clj

## ansible-core.clj
(ns ansible.core
  (:require [clojure.java.shell :as sh]
            [clojure.string :as str]
            [cheshire.core :as cheshire]
            [clojure.java.io :as io])
  (:import (java.io File)
           (com.fasterxml.jackson.core JsonGenerator)
           (java.util Base64)
           (java.security MessageDigest MessageDigest$Delegate)))

(defmacro with-temp-files
  "Create a block where `varname` is a temporary `File` containing `content`.
   The tempfile is deleted right after the body is run!"
  [bindings & body]
  (let [bindings (partition 2 bindings)]
    `(let ~(into []
                 (mapcat
                   (fn [[bind-name content]]
                     `[~bind-name (File/createTempFile ~(str bind-name) ".json")]))
                 bindings)
       ~@(mapv
           (fn [[bind-name content]]
             `(io/copy ~content ~bind-name))
           bindings)
       (let [rv# (do ~@body)]
         ~@(mapv
             (fn [[bind-name _]]
               `(.delete ^File ~bind-name))
             bindings)
         rv#))))

(def ^MessageDigest$Delegate md5-encoder (MessageDigest/getInstance "MD5"))

(defn hash-for-string
  "Returns a hash for a given string"
  [^String s]
  (.encodeToString (Base64/getUrlEncoder) (.digest md5-encoder (.getBytes s))))

(defn persistent-file
  "Creates a temporary file that won't be deleted and returns the filename string"
  [content]
  (let [hash (hash-for-string content)
        file (io/file (str (System/getProperty "java.io.tmpdir") "/ansible-" hash))]
    (when-not (.exists file)
      (io/copy content file))
    (str file)))

(defn emit-ini-value
  [v]
  (cond (sequential? v) (str/join " " (mapv emit-ini-value v))
        (keyword? v) (name v)
        :else v))

(defn emit-ini-section
  [m]
  (reduce
    (fn [s [k v]]
      (reduce
        (fn [s v]
          (str s \newline (name k) \= (emit-ini-value v)))
        s (cond (sequential? v) v
                (map? v) (mapv (fn [[k v]] (str (name k) \= (emit-ini-value v)))
                               (sort-by first v))
                :else [v])))
    "" (if (map? m) (sort-by first m) m)))


(defn ini-str
  "Creates an ini-like string for the given data.
  Works well with systemd service configs"
  ([d] (ini-str d nil))
  ([d note]
   {:pre [(map? d)]}
   (reduce-kv
     (fn [s sec m]
       (str s \newline \[ (name sec) \] (emit-ini-section m) \newline))
     (if note (str "# " note) "")
     d)))

(defn kvs
  "Generates a string like 'k0=v0 k1=v1' that ansible sometimes wants."
  [m]
  (str/join " "
            (mapv (fn [[k v]]
                    (str (name k) "=" (if (keyword? v) (name v) v)))
                  m)))

(defn json
  "Generates a json string where any keywords are converted with a simple (name x)"
  [x]
  (let [orig (find-protocol-impl cheshire.generate/JSONable :key)
        _ (extend clojure.lang.Keyword
            cheshire.generate/JSONable
            {:to-json (fn encode-named
                        [^clojure.lang.Keyword k ^JsonGenerator jg]
                        (.writeString jg (name k)))})
        res (cheshire/generate-string x {:pretty true
                                         :key-fn name})]
    (extend clojure.lang.Keyword cheshire.generate/JSONable orig)
    res))
#_(json {:a/b :b/c})

(defn nlsv
  "New line separated values"
  [& args]
  (str/join "\n" args))

(defn ssv
  "Space separted values"
  [& vals]
  (str/join " " vals))

(defn path-join
  "A robust path joiner, adds a slash between any argument"
  [& args]
  (let [sep (File/separator)
        sws? #(.startsWith ^String % sep)
        ews? #(.endsWith ^String % sep)
        join (fn [a b]
               (if (ews? a)
                 (if (sws? b) (str a (subs b 1)) (str a b))
                 (if (sws? b) (str a b) (str a sep b))))]
    (reduce join args)))
#_(path-join "a/" "/b/" "/c/" "/foo.bar")

(defn dir-exists? [dir] (.exists (io/file dir)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ANSIBLE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn lookup-file
  "A {{lookup('file', ...}} for usage in playbook values"
  [f]
  (str "{{ lookup('file', '" f "') }}"))

(defn parse-out
  "Parses ansible output. Could be made smarter by finding valid json within
  output and parsing it..."
  [out]
  (try
    (cheshire/parse-string out)
    (catch Exception _
      (mapv str/trim (str/split out #"\n")))))

(defn gen-ansible-files
  [work-dir inventory config]
  {:pre [(dir-exists? work-dir)]}
  (spit (path-join work-dir "hosts.json") (json inventory))
  (spit (path-join work-dir "ansible.cfg") (ini-str config)))

(defn ansible-sh!
  [& args]
  (let [start (System/currentTimeMillis)
        res (apply sh/sh args)
        end (System/currentTimeMillis)]
    (-> res
        (assoc :took (/ (- end start) 1000.0))
        (update :out parse-out)
        (with-meta {:cmd (vec args)}))))

(defn ansible
  "Runs ansible with the given arguments"
  [{:keys [work-dir inventory config vars exec]} host-pattern & args]
  (gen-ansible-files work-dir inventory config)
  (apply ansible-sh! (or exec "ansible") host-pattern
         "-i" "hosts.json"
         "-e" (json vars)
         (concat args [:dir work-dir])))

(defn ansible-play
  "Runs ansible-playbook with the given arguments"
  [{:keys [work-dir inventory config vars exec
           play-file]} playbook & args]
  (gen-ansible-files work-dir inventory config)
  (let [run (fn [playb-file]
              (apply ansible-sh! (or exec "ansible-playbook")
                     "-i" "hosts.json"
                     "-e" (json vars)
                     (concat args [(str playb-file)] [:dir work-dir])))]
    (if (some? play-file)
      (let [play-file (str (name play-file) ".yaml")]
        (spit (path-join work-dir play-file) (json playbook))
        (run play-file))
      (with-temp-files [playb-file (json playbook)]
        (run playb-file)))))

## example.clj
(ns ansible.your.example
  (:require
    [clojure.java.io :as io]
    [clojure.java.shell :as sh]
    [ansible.core :refer [ansible ansible-play ssv path-join kvs ini-str nlsv json
                          lookup-file]]
    [nomad :as nomad]))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MY ANSIBLE CFG
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn systemd-file [s] (str "/etc/systemd/system/" (name s) ".service"))

(defn datomic-dir [append]
  (str "/usr/local/datomic/" append))

(defn prod-cfg
  [& kork]
  (get-in
    (nomad/with-location-override
      {:environment "prod"
       :hostname "example.com"}
      (nomad/read-config (io/resource "config/main.edn")))
    kork))

;; Fill in PW here and run in repl to get PW into memory:
#_(def sudo-password "")
(defonce sudo-password "")
(def ssh-port 2291)
(def ansible-common-args
  ["--become"
   "--become-user" "root"
   "--forks" "10"
   "--ssh-common-args" (ssv "-o" "ControlMaster=auto"
                            "-o" "ControlPath=\"/home/rauh/.ssh/%l-%h-%p-%r\""
                            "-o" "ControlPersist=24h")])
(def inventory {:hosts/openresty {:hosts {"8.8.8.8" ""}}
                :hosts/app {:hosts {"8.9.223.999" ""}}
                :hosts/cassandra {:hosts {"cassandra0" ""
                                          "cassandra1" ""}}})
(def ansible-cfg {"defaults" {;"stdout_callback" "json";; VERY verbose output!
                              "remote_port" ssh-port
                              "fact_caching" "jsonfile"
                              "fact_caching_connection" "/tmp/ansible-fact-cache/"}
                  "ssh_connection" {"pipelining" "True"}})
(def vars {"ansible_become_pass" sudo-password})
(def ansible-work-dir "ansible-cfg")

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Create wrappers around the two main command:
(defn ansible*
  [hosts & args]
  (apply ansible {:work-dir ansible-work-dir
                  :inventory inventory
                  :config ansible-cfg
                  :vars vars}
         (name hosts)
         (concat ansible-common-args args)))

(defn ansible-play*
  [playbook & [maybe-playbook & actual-args :as args]]
  (let [playfile? (or (keyword? playbook)
                      (string? playbook))]
    (apply ansible-play {:work-dir ansible-work-dir
                         :inventory inventory
                         :config ansible-cfg
                         :vars vars
                         :play-file (when playfile? playbook)}
           (mapv (partial merge {:gather_facts false}) (if playfile? maybe-playbook playbook))
           (concat ansible-common-args (if playfile? actual-args args)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ANSIBLE COMMANDS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(comment
  (ansible* "all" "--list-hosts")
  (ansible* :hosts/openresty "-a" "/usr/bin/whoami")
  (ansible* :hosts/openresty "-a" "which lein")
  (ansible* :hosts/openresty "-m" "setup")
  (ansible* :hosts/openresty "-m" "ping")
  (ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:daemon_reload true}))
  ;; Start app:
  (ansible* :hosts/app "-m" "systemd" "-a" (kvs {:name :systemd/clj-app :state :started}))
  ;; Datomic license info:
  (ansible* :hosts/datomic "-a" (ssv (datomic-dir "bin/datomic") "describe-license"
                                     (datomic-config-file)))
  (ansible* :hosts/elasticsearch "-a" (ssv "bash" "-c" "'netstat -ln | grep :9200'"))
  (ansible* :hosts/elasticsearch "-a" (ssv "systemctl" "status" "elasticsearch"))
  ;; Reload openresty:
  (ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:name :systemd/openresty :state :reloaded})))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def start-stop-daemon "-/sbin/start-stop-daemon")
(def ansbile-managed "Ansible managed")

(defn rsync-copy
  "A :block which rsync's in two steps since rsync task cant sudo w/ password:
  1. Rsync to the given temp dir (retained so rsync is still fast)
  2. Rsync within the server to dest"
  [{:keys [name dest temp src]} & [opt-to-remote opt-on-remote]]
  {:block [{:name (str name ": To remote")
            :synchronize (merge {:delete true
                                 :dest temp
                                 :src src
                                 :group false
                                 :owner false
                                 :links false}
                                opt-to-remote)
            ;; We can't sudo w/ rsync:
            :become false}
           {:name (str name ": Within remote")
            :synchronize (merge {:dest dest
                                 :group false
                                 :owner false
                                 :perms false
                                 :src temp
                                 :links false}
                                opt-on-remote)
            ;; To rsync within the remote host
            :delegate_to "{{ inventory_hostname }}"}]})

(defn std-file-perms
  "Recursively sets the perms to 0775/0664 on the given dir.
   Changes all files to user/group"
  [dir user group]
  {:block [{:name (str "Set perms to 0775/0664 for " dir)
            :command (ssv "chmod"
                          "-c" ;; output changed files
                          "-R" ;; recursive
                          "ug=rw,o=r,a-x+X"
                          dir)
            :register "chmod_status"
            :changed_when "chmod_status.stdout != \"\""}
           {:name (str "chown: " user ":" group)
            :file {:group group
                   :owner user
                   :path dir
                   :state "directory"
                   :recurse true}}]})

(defn enable-and-start-service
  ([svc] (enable-and-start-service svc :state/started))
  ([svc state]
   {:name (str "Enabling " (name svc) " service")
    :service {:name svc
              :state state
              :enabled true}}))

(defn enable-and-start-serviced
  ([svc] (enable-and-start-serviced svc :state/started))
  ([svc state]
   {:name (str "Enabling " (name svc) " service")
    :systemd {:name svc
              :state state
              :enabled true}}))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; MY PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def all-handlers
  (let [restart (fn [handler service]
                  {:name handler
                   :systemd {:name service
                             :state :state/restarted}})]
    [{:name :handler/reload-systemd
      :systemd {:daemon_reload true}}
     {:name :handler/reload-openresty
      :systemd {:name :systemd/openresty
                :state :state/reloaded}}
     (restart :handler/restart-redis-stats :systemd/redis-stats)
     (restart :handler/restart-redis-volatile :systemd/redis-volatile)
     (restart :handler/restart-datadog :systemd/datadog-agent)]))

(defn copy-openresty-systemd
  []
  {:block [{:name "Copy OpenResty systemd service file"
            :copy {:content (ini-str
                              (let [bin "/usr/local/openresty/bin/openresty"
                                    pid "/run/nginx.pid"
                                    g-conf "'daemon on; master_process on;'"]
                                {:Unit {:Description "Openresty server"
                                        :After :network.target}
                                 :Service {:Type :type/forking
                                           ;; Note: User/group is set by the config file
                                           #_#_:ExecStartPre [[bin "-t" "-q" "-g" g-conf]]
                                           :PIDFile pid
                                           :ExecStart [[bin "-g" g-conf]]
                                           :ExecReload [[bin "-g" g-conf "-s" "reload"]]
                                           :ExecStop [[start-stop-daemon "--quiet" "--stop"
                                                       "--retry" "QUIT/5" "--pidfile" pid]]
                                           :TimeoutStopSec 5
                                           :PrivateTmp true
                                           :KillMode "mixed"}
                                 :Install {:WantedBy :multi-user.target}})
                              ansbile-managed)
                   :dest (systemd-file :systemd/openresty)
                   :mode "0644"}
            :notify [:handler/reload-systemd]}
           {:name "Enabling openresty service"
            :systemd {:name :systemd/openresty
                      :enabled true}}]})

(defn datomic-dir [append]
  (str "/usr/local/datomic/" append))
(defn datomic-config-file []
  (datomic-dir "transactor-cass-prod.properties"))

(defn install-datomic-systemd
  []
  {:block [{:name "Copy Datomic systemd service file:"
            :copy {:content (ini-str
                              (let [bin "/usr/local/datomic/bin/transactor"
                                    data-dir (datomic-dir "data/")
                                    ug "datomic:datomic"
                                    log-dir (datomic-dir "log/")]
                                {:Unit {:Description "SRS Datomic"
                                        :After [[:network.target "cassandra.service"]]}
                                 :Service {:Type :type/simple
                                           :User "datomic"
                                           :Group "datomic"
                                           :PermissionsStartOnly true
                                           :ExecStartPre [["-/bin/mkdir" "-p" data-dir]
                                                          ["-/bin/chown" "-R" ug data-dir]
                                                          ["-/bin/mkdir" "-p" log-dir]
                                                          ["-/bin/chown" ug log-dir]]
                                           :ExecStart [[bin
                                                        "-Xms2048m" "-Xmx2048m"
                                                        (datomic-config-file)]]
                                           :Restart "always"
                                           :RestartSecs 3}
                                 :Install {:WantedBy :multi-user.target}})
                              ansbile-managed)
                   :dest (systemd-file :systemd/datomic)
                   :mode "0644"}
            :notify [:handler/reload-systemd]}
           {:name "Enabling datomic service systemd"
            :systemd {:name :systemd/datomic
                      :enabled true}}]})

(defn create-backup-user
  []
  {:name "Creating backup user"
   :user {:name "backup-user"
          :generate_ssh_key true}})

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; RUN PLAYBOOKS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(comment
  (ansible-play*
    :playbook/backup
    [{:hosts :hosts/backup
      :tasks [(create-backup-user)]
      :handlers all-handlers}])

  ;; CASSANDRA ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  (ansible-play*
    [{:hosts :hosts/cassandra
      :tasks [(enable-and-start-service :service/cassandra)]
      :handlers all-handlers}])
  ;; DATOMIC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  (ansible-play*
    [{:hosts :hosts/datomic
      :tasks [(install-datomic-systemd)]
      :handlers all-handlers}]
    #_"--check")
  ;; OPENRESTY ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  (ansible-play*
    [{:hosts :hosts/openresty
      :tasks [(copy-openresty-systemd)]
      :handlers all-handlers}]
    #_"--check"))
	(ns ansible.core
	(:require [clojure.java.shell :as sh]
	[clojure.string :as str]
	[cheshire.core :as cheshire]
	[clojure.java.io :as io])
	(:import (java.io File)
	(com.fasterxml.jackson.core JsonGenerator)
	(java.util Base64)
	(java.security MessageDigest MessageDigest$Delegate)))

	(defmacro with-temp-files
	"Create a block where `varname` is a temporary `File` containing `content`.
	The tempfile is deleted right after the body is run!"
	[bindings & body]
	(let [bindings (partition 2 bindings)]
	`(let ~(into []
	(mapcat
	(fn [[bind-name content]]
	`[~bind-name (File/createTempFile ~(str bind-name) ".json")]))
	bindings)
	~@(mapv
	(fn [[bind-name content]]
	`(io/copy ~content ~bind-name))
	bindings)
	(let [rv# (do ~@body)]
	~@(mapv
	(fn [[bind-name _]]
	`(.delete ^File ~bind-name))
	bindings)
	rv#))))

	(def ^MessageDigest$Delegate md5-encoder (MessageDigest/getInstance "MD5"))

	(defn hash-for-string
	"Returns a hash for a given string"
	[^String s]
	(.encodeToString (Base64/getUrlEncoder) (.digest md5-encoder (.getBytes s))))

	(defn persistent-file
	"Creates a temporary file that won't be deleted and returns the filename string"
	[content]
	(let [hash (hash-for-string content)
	file (io/file (str (System/getProperty "java.io.tmpdir") "/ansible-" hash))]
	(when-not (.exists file)
	(io/copy content file))
	(str file)))

	(defn emit-ini-value
	[v]
	(cond (sequential? v) (str/join " " (mapv emit-ini-value v))
	(keyword? v) (name v)
	:else v))

	(defn emit-ini-section
	[m]
	(reduce
	(fn [s [k v]]
	(reduce
	(fn [s v]
	(str s \newline (name k) \= (emit-ini-value v)))
	s (cond (sequential? v) v
	(map? v) (mapv (fn [[k v]] (str (name k) \= (emit-ini-value v)))
	(sort-by first v))
	:else [v])))
	"" (if (map? m) (sort-by first m) m)))


	(defn ini-str
	"Creates an ini-like string for the given data.
	Works well with systemd service configs"
	([d] (ini-str d nil))
	([d note]
	{:pre [(map? d)]}
	(reduce-kv
	(fn [s sec m]
	(str s \newline \[ (name sec) \] (emit-ini-section m) \newline))
	(if note (str "# " note) "")
	d)))

	(defn kvs
	"Generates a string like 'k0=v0 k1=v1' that ansible sometimes wants."
	[m]
	(str/join " "
	(mapv (fn [[k v]]
	(str (name k) "=" (if (keyword? v) (name v) v)))
	m)))

	(defn json
	"Generates a json string where any keywords are converted with a simple (name x)"
	[x]
	(let [orig (find-protocol-impl cheshire.generate/JSONable :key)
	_ (extend clojure.lang.Keyword
	cheshire.generate/JSONable
	{:to-json (fn encode-named
	[^clojure.lang.Keyword k ^JsonGenerator jg]
	(.writeString jg (name k)))})
	res (cheshire/generate-string x {:pretty true
	:key-fn name})]
	(extend clojure.lang.Keyword cheshire.generate/JSONable orig)
	res))
	#_(json {:a/b :b/c})

	(defn nlsv
	"New line separated values"
	[& args]
	(str/join "\n" args))

	(defn ssv
	"Space separted values"
	[& vals]
	(str/join " " vals))

	(defn path-join
	"A robust path joiner, adds a slash between any argument"
	[& args]
	(let [sep (File/separator)
	sws? #(.startsWith ^String % sep)
	ews? #(.endsWith ^String % sep)
	join (fn [a b]
	(if (ews? a)
	(if (sws? b) (str a (subs b 1)) (str a b))
	(if (sws? b) (str a b) (str a sep b))))]
	(reduce join args)))
	#_(path-join "a/" "/b/" "/c/" "/foo.bar")

	(defn dir-exists? [dir] (.exists (io/file dir)))

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; ANSIBLE
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(defn lookup-file
	"A {{lookup('file', ...}} for usage in playbook values"
	[f]
	(str "{{ lookup('file', '" f "') }}"))

	(defn parse-out
	"Parses ansible output. Could be made smarter by finding valid json within
	output and parsing it..."
	[out]
	(try
	(cheshire/parse-string out)
	(catch Exception _
	(mapv str/trim (str/split out #"\n")))))

	(defn gen-ansible-files
	[work-dir inventory config]
	{:pre [(dir-exists? work-dir)]}
	(spit (path-join work-dir "hosts.json") (json inventory))
	(spit (path-join work-dir "ansible.cfg") (ini-str config)))

	(defn ansible-sh!
	[& args]
	(let [start (System/currentTimeMillis)
	res (apply sh/sh args)
	end (System/currentTimeMillis)]
	(-> res
	(assoc :took (/ (- end start) 1000.0))
	(update :out parse-out)
	(with-meta {:cmd (vec args)}))))

	(defn ansible
	"Runs ansible with the given arguments"
	[{:keys [work-dir inventory config vars exec]} host-pattern & args]
	(gen-ansible-files work-dir inventory config)
	(apply ansible-sh! (or exec "ansible") host-pattern
	"-i" "hosts.json"
	"-e" (json vars)
	(concat args [:dir work-dir])))

	(defn ansible-play
	"Runs ansible-playbook with the given arguments"
	[{:keys [work-dir inventory config vars exec
	play-file]} playbook & args]
	(gen-ansible-files work-dir inventory config)
	(let [run (fn [playb-file]
	(apply ansible-sh! (or exec "ansible-playbook")
	"-i" "hosts.json"
	"-e" (json vars)
	(concat args [(str playb-file)] [:dir work-dir])))]
	(if (some? play-file)
	(let [play-file (str (name play-file) ".yaml")]
	(spit (path-join work-dir play-file) (json playbook))
	(run play-file))
	(with-temp-files [playb-file (json playbook)]
	(run playb-file)))))
	(ns ansible.your.example
	(:require
	[clojure.java.io :as io]
	[clojure.java.shell :as sh]
	[ansible.core :refer [ansible ansible-play ssv path-join kvs ini-str nlsv json
	lookup-file]]
	[nomad :as nomad]))

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; MY ANSIBLE CFG
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(defn systemd-file [s] (str "/etc/systemd/system/" (name s) ".service"))

	(defn datomic-dir [append]
	(str "/usr/local/datomic/" append))

	(defn prod-cfg
	[& kork]
	(get-in
	(nomad/with-location-override
	{:environment "prod"
	:hostname "example.com"}
	(nomad/read-config (io/resource "config/main.edn")))
	kork))

	;; Fill in PW here and run in repl to get PW into memory:
	#_(def sudo-password "")
	(defonce sudo-password "")
	(def ssh-port 2291)
	(def ansible-common-args
	["--become"
	"--become-user" "root"
	"--forks" "10"
	"--ssh-common-args" (ssv "-o" "ControlMaster=auto"
	"-o" "ControlPath=\"/home/rauh/.ssh/%l-%h-%p-%r\""
	"-o" "ControlPersist=24h")])
	(def inventory {:hosts/openresty {:hosts {"8.8.8.8" ""}}
	:hosts/app {:hosts {"8.9.223.999" ""}}
	:hosts/cassandra {:hosts {"cassandra0" ""
	"cassandra1" ""}}})
	(def ansible-cfg {"defaults" {;"stdout_callback" "json";; VERY verbose output!
	"remote_port" ssh-port
	"fact_caching" "jsonfile"
	"fact_caching_connection" "/tmp/ansible-fact-cache/"}
	"ssh_connection" {"pipelining" "True"}})
	(def vars {"ansible_become_pass" sudo-password})
	(def ansible-work-dir "ansible-cfg")

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; Create wrappers around the two main command:
	(defn ansible*
	[hosts & args]
	(apply ansible {:work-dir ansible-work-dir
	:inventory inventory
	:config ansible-cfg
	:vars vars}
	(name hosts)
	(concat ansible-common-args args)))

	(defn ansible-play*
	[playbook & [maybe-playbook & actual-args :as args]]
	(let [playfile? (or (keyword? playbook)
	(string? playbook))]
	(apply ansible-play {:work-dir ansible-work-dir
	:inventory inventory
	:config ansible-cfg
	:vars vars
	:play-file (when playfile? playbook)}
	(mapv (partial merge {:gather_facts false}) (if playfile? maybe-playbook playbook))
	(concat ansible-common-args (if playfile? actual-args args)))))

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; ANSIBLE COMMANDS
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(comment
	(ansible* "all" "--list-hosts")
	(ansible* :hosts/openresty "-a" "/usr/bin/whoami")
	(ansible* :hosts/openresty "-a" "which lein")
	(ansible* :hosts/openresty "-m" "setup")
	(ansible* :hosts/openresty "-m" "ping")
	(ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:daemon_reload true}))
	;; Start app:
	(ansible* :hosts/app "-m" "systemd" "-a" (kvs {:name :systemd/clj-app :state :started}))
	;; Datomic license info:
	(ansible* :hosts/datomic "-a" (ssv (datomic-dir "bin/datomic") "describe-license"
	(datomic-config-file)))
	(ansible* :hosts/elasticsearch "-a" (ssv "bash" "-c" "'netstat -ln \| grep :9200'"))
	(ansible* :hosts/elasticsearch "-a" (ssv "systemctl" "status" "elasticsearch"))
	;; Reload openresty:
	(ansible* :hosts/openresty "-m" "systemd" "-a" (kvs {:name :systemd/openresty :state :reloaded})))

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; PLAYBOOKS
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(def start-stop-daemon "-/sbin/start-stop-daemon")
	(def ansbile-managed "Ansible managed")

	(defn rsync-copy
	"A :block which rsync's in two steps since rsync task cant sudo w/ password:
	1. Rsync to the given temp dir (retained so rsync is still fast)
	2. Rsync within the server to dest"
	[{:keys [name dest temp src]} & [opt-to-remote opt-on-remote]]
	{:block [{:name (str name ": To remote")
	:synchronize (merge {:delete true
	:dest temp
	:src src
	:group false
	:owner false
	:links false}
	opt-to-remote)
	;; We can't sudo w/ rsync:
	:become false}
	{:name (str name ": Within remote")
	:synchronize (merge {:dest dest
	:group false
	:owner false
	:perms false
	:src temp
	:links false}
	opt-on-remote)
	;; To rsync within the remote host
	:delegate_to "{{ inventory_hostname }}"}]})

	(defn std-file-perms
	"Recursively sets the perms to 0775/0664 on the given dir.
	Changes all files to user/group"
	[dir user group]
	{:block [{:name (str "Set perms to 0775/0664 for " dir)
	:command (ssv "chmod"
	"-c" ;; output changed files
	"-R" ;; recursive
	"ug=rw,o=r,a-x+X"
	dir)
	:register "chmod_status"
	:changed_when "chmod_status.stdout != \"\""}
	{:name (str "chown: " user ":" group)
	:file {:group group
	:owner user
	:path dir
	:state "directory"
	:recurse true}}]})

	(defn enable-and-start-service
	([svc] (enable-and-start-service svc :state/started))
	([svc state]
	{:name (str "Enabling " (name svc) " service")
	:service {:name svc
	:state state
	:enabled true}}))

	(defn enable-and-start-serviced
	([svc] (enable-and-start-serviced svc :state/started))
	([svc state]
	{:name (str "Enabling " (name svc) " service")
	:systemd {:name svc
	:state state
	:enabled true}}))

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; MY PLAYBOOKS
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(def all-handlers
	(let [restart (fn [handler service]
	{:name handler
	:systemd {:name service
	:state :state/restarted}})]
	[{:name :handler/reload-systemd
	:systemd {:daemon_reload true}}
	{:name :handler/reload-openresty
	:systemd {:name :systemd/openresty
	:state :state/reloaded}}
	(restart :handler/restart-redis-stats :systemd/redis-stats)
	(restart :handler/restart-redis-volatile :systemd/redis-volatile)
	(restart :handler/restart-datadog :systemd/datadog-agent)]))

	(defn copy-openresty-systemd
	[]
	{:block [{:name "Copy OpenResty systemd service file"
	:copy {:content (ini-str
	(let [bin "/usr/local/openresty/bin/openresty"
	pid "/run/nginx.pid"
	g-conf "'daemon on; master_process on;'"]
	{:Unit {:Description "Openresty server"
	:After :network.target}
	:Service {:Type :type/forking
	;; Note: User/group is set by the config file
	#_#_:ExecStartPre [[bin "-t" "-q" "-g" g-conf]]
	:PIDFile pid
	:ExecStart [[bin "-g" g-conf]]
	:ExecReload [[bin "-g" g-conf "-s" "reload"]]
	:ExecStop [[start-stop-daemon "--quiet" "--stop"
	"--retry" "QUIT/5" "--pidfile" pid]]
	:TimeoutStopSec 5
	:PrivateTmp true
	:KillMode "mixed"}
	:Install {:WantedBy :multi-user.target}})
	ansbile-managed)
	:dest (systemd-file :systemd/openresty)
	:mode "0644"}
	:notify [:handler/reload-systemd]}
	{:name "Enabling openresty service"
	:systemd {:name :systemd/openresty
	:enabled true}}]})

	(defn datomic-dir [append]
	(str "/usr/local/datomic/" append))
	(defn datomic-config-file []
	(datomic-dir "transactor-cass-prod.properties"))

	(defn install-datomic-systemd
	[]
	{:block [{:name "Copy Datomic systemd service file:"
	:copy {:content (ini-str
	(let [bin "/usr/local/datomic/bin/transactor"
	data-dir (datomic-dir "data/")
	ug "datomic:datomic"
	log-dir (datomic-dir "log/")]
	{:Unit {:Description "SRS Datomic"
	:After [[:network.target "cassandra.service"]]}
	:Service {:Type :type/simple
	:User "datomic"
	:Group "datomic"
	:PermissionsStartOnly true
	:ExecStartPre [["-/bin/mkdir" "-p" data-dir]
	["-/bin/chown" "-R" ug data-dir]
	["-/bin/mkdir" "-p" log-dir]
	["-/bin/chown" ug log-dir]]
	:ExecStart [[bin
	"-Xms2048m" "-Xmx2048m"
	(datomic-config-file)]]
	:Restart "always"
	:RestartSecs 3}
	:Install {:WantedBy :multi-user.target}})
	ansbile-managed)
	:dest (systemd-file :systemd/datomic)
	:mode "0644"}
	:notify [:handler/reload-systemd]}
	{:name "Enabling datomic service systemd"
	:systemd {:name :systemd/datomic
	:enabled true}}]})

	(defn create-backup-user
	[]
	{:name "Creating backup user"
	:user {:name "backup-user"
	:generate_ssh_key true}})

	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	;; RUN PLAYBOOKS
	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(comment
	(ansible-play*
	:playbook/backup
	[{:hosts :hosts/backup
	:tasks [(create-backup-user)]
	:handlers all-handlers}])

	;; CASSANDRA ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(ansible-play*
	[{:hosts :hosts/cassandra
	:tasks [(enable-and-start-service :service/cassandra)]
	:handlers all-handlers}])
	;; DATOMIC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(ansible-play*
	[{:hosts :hosts/datomic
	:tasks [(install-datomic-systemd)]
	:handlers all-handlers}]
	#_"--check")
	;; OPENRESTY ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	(ansible-play*
	[{:hosts :hosts/openresty
	:tasks [(copy-openresty-systemd)]
	:handlers all-handlers}]
	#_"--check"))