Freistilbox utils

freistilbox-deploy-check.py

#!/usr/bin/env python3

import os
import sys
import re
import argparse
from datetime import timedelta
from datetime import datetime


deploy_log_line_pattern = re.compile(r'^[A-Z], \[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})#(\d+)\] \s*([\w]+) \S+ : (.+)$')

job_success_msg = 'Deployment finished.'
job_fail_msg = 'Aborting deployment.'

JOB_STATUS_RUNNING = 0
JOB_STATUS_SUCCEEDED = 1
JOB_STATUS_FAILED = 2
JOB_STATUS_LABEL = {
  JOB_STATUS_RUNNING: 'running',
  JOB_STATUS_FAILED: 'fail',
  JOB_STATUS_SUCCEEDED: 'success'
}



def read_file(file_path, continuous=False):

  if not continuous:
    with open(file_path,'r') as file_handle:
      for line in file_handle:
        line = line.strip()
        if line:
          yield line
  else:
    with open(file_path,'r') as file_handle:
      while not file_handle.closed:
        line = file_handle.readline().strip()
        if line:
          yield line


class ParsingError(Exception):
  def __init__(self, raw_line):
        super(ParsingError, self).__init__("Parsing error!! Raw input line: %s" % raw_line)


class DeployJob:
  
  def __init__(self, job_id, status, begin, end, log_entries):
    """
    
    """
    self.job_id = job_id
    self.status = status
    self.begin = begin
    self.end = end
    self.log = log_entries
    
  
  @property
  def completed(self) -> bool:
    return self.status in [JOB_STATUS_FAILED, JOB_STATUS_SUCCEEDED]
  
  @property
  def duration(self) -> timedelta:
    if self.begin and self.end:
      return self.end - self.begin
    else:
      return None


def read_last_job(file_path, since=None, commit_hash=None, continuous=False):

  last_job_id = None
  last_job_log_entries = None
  last_job_status = None
  last_job_begin = None
  last_job_end = None
  last_job_commit_hash_found = False

  for line in read_file(file_path, continuous):
    
    if not line or line.startswith('#'):
      # skip empty or commented lines
      continue
    parsed_line = deploy_log_line_pattern.match(line)
    if not parsed_line:
      raise ParsingError(line)
    
    log_time = datetime.strptime(parsed_line.group(1), r'%Y-%m-%d %H:%M:%S')
    job_id = parsed_line.group(2)
    log_level = parsed_line.group(3)
    log_msg = parsed_line.group(4)
    
    if since and log_time < since:
      # skip entry if it's out of scope
      continue
    
    if job_id != last_job_id:
      # found a new job
      last_job_id = job_id
      last_job_log_entries = []
      last_job_status = JOB_STATUS_RUNNING
      last_job_begin = log_time
    
    last_job_log_entries.append({'time': log_time, 'level': log_level, 'msg': log_msg})
    
    if log_msg == job_fail_msg:
      last_job_status = JOB_STATUS_FAILED
      last_job_end = log_time
      if last_job_commit_hash_found or (not commit_hash and continuous):
        break;
    elif log_msg == job_success_msg:
      last_job_status = JOB_STATUS_SUCCEEDED
      last_job_end = log_time
      if last_job_commit_hash_found or (not commit_hash and continuous):
        break;
    elif commit_hash and log_msg.startswith("Current release is %s" % commit_hash):
      last_job_commit_hash_found = True
    
  if not last_job_id:
    return None
  elif commit_hash and not last_job_commit_hash_found:
    return None
  else:
    return DeployJob(last_job_id, last_job_status, last_job_begin, last_job_end, last_job_log_entries)



if __name__ == "__main__":
  
  parser = argparse.ArgumentParser(description='Freistilbox deployment log interpreter.')
  parser.add_argument('--file', '-f', default='~/.deploy/deploy.log', help='Path to log file.')
  parser.add_argument('--quiet', '-q', action='store_true', help='Don\'t write to stdout.')
  parser.add_argument('--verbose', '-v', action='store_true', help='Show job details. (Ignored if --quiet is set)')
  parser.add_argument('--since', '-s', type=int, metavar='TIMESTAMP', help='Don\'t interpret log entries before unix timestamp.')
  parser.add_argument('--commit', '-c', metavar='HASH', help='Look for job triggered by specific git commit.')
  parser.add_argument('--wait', '-w', action='store_true', help='Keep log open until last job completes. This is ignored if last job already completed.')
  # TODO: add wait-timeout argument (and an implementation for it)

  args = parser.parse_args()
  
  file_path = os.path.expanduser(args.file)
  if not os.path.exists(file_path):
    parser.error("File not found: %s" % file_path)
  
  if args.quiet:
    def stdout(msg=""):
      pass
  else:
    def stdout(msg=""):
      print(msg)
  def stderr(msg):
    print(msg, file=sys.stderr)
  
  try:
    
    since = None
    if args.since:
      since = datetime.fromtimestamp(args.since)
    
    job = read_last_job(file_path, since, args.commit)
    
    if args.wait:
      if args.commit and job and job.completed:
        # Already got the completed job triggered by args.commit.
        # No need to wait any longer - continue right away!
        pass
      elif job and job.completed:
        # Got time of last completed job in past.
        # Now wait for next job and continue once it wrote its "job complete" message into the log file.
        job = read_last_job(file_path, job.end + timedelta(microseconds=1), None, True)
      else:
        # Last job in past did not yet complete (or no job found at all since log file was empty).
        # Wait for the next "job complete" message in the log file and then continue.
        job = read_last_job(file_path, since, args.commit, True)

    if not job:
      msg = "No deployment job found"
      if args.commit:
        msg += " for git commit %s" % args.commit
      if since:
        msg += " since %s" % since.strftime("%Y-%m-%d %H:%M:%S")
      stderr("%s." % msg)
      sys.exit(23)

    if args.verbose:
      if since:
        stdout("Checked log entries since %s." % since.strftime("%Y-%m-%d %H:%M:%S"))
      stdout()
      if args.commit:
        stdout("Job triggered by git commit %s" % args.commit)
      else:
        stdout("Most recent job")
      stdout("  id: %s" % job.job_id)
      stdout("  status: %s" % JOB_STATUS_LABEL.get(job.status, 'unknown').upper())
      if job.completed:
        since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
        till = job.end.strftime("%H:%M:%S")
        stdout("  from %s to %s (%s)" % (since, till, job.duration))
      else:
        since = job.begin.strftime("%Y-%m-%d %H:%M:%S")
        duration = (datetime.now() - job.begin)
        stdout("  running since %s (%s)" % (since, duration))
      stdout()
    
    for log_entry in job.log:
      stdout("%s - %s: %s" % (log_entry['time'].strftime('%c'), log_entry['level'], log_entry['msg']))

    if job.status == JOB_STATUS_SUCCEEDED:
      sys.exit(0)
    elif job.status == JOB_STATUS_RUNNING:
      sys.exit(21)
    elif job.status == JOB_STATUS_FAILED:
      sys.exit(22)
    else:
      sys.exit(24)
  
  except PermissionError as e:
    stderr(e)
    sys.exit(1)
  except ParsingError as e:
    stderr(e)
    sys.exit(13)
  except KeyboardInterrupt as e:
    stdout()
    sys.exit(1)

freistilbox-mysql-dump.sh

#!/bin/bash

set -e
set -o pipefail


[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox cluster ID defined."; exit 1; }
CLUSTER=$1
shift

[ "$#" -gt 0 ] || { >&2 echo "no Freistilbox site ID defined."; exit 1; }
SITE=$1
shift

SSH_HOST="${CLUSTER}s.freistilbox.net"

DRUSH_CMD="/usr/local/bin/drush"
FSB_DOCROOT="/srv/www/freistilbox/clients/c11000/${SITE}/current/docroot/"
DUMP_OPTIONS="--extra-dump=--opt --extra-dump=--hex-blob --extra-dump=--skip-comments"

echo "Creating MySQL Dump of site ${SITE} on Freistilbox cluster ${CLUSTER}"

ssh -C ${SITE}@${SSH_HOST} "${DRUSH_CMD} --root=$FSB_DOCROOT sql:dump $DUMP_OPTIONS" | gzip --no-name > ${SITE}_dump.sql.gz