Useful scripts

a-useful scripts.md

scripts about cmd, github gh cli ...

git.sh

# move or remove them before you merge, use this to get related files
git xxx 2>&1|grep -E '^\s'|cut -f2-|xargs -I {} echo "{}"

rebuild.sh

#!/bin/bash
set -ex
# /Users/ark
PWD=$(pwd)
WORK_DIR="$PWD"/Code/OpenMLDB
pushd "$WORK_DIR"
time=$(date +%Y-%m-%d\ %H:%M:%S)
echo "start rebuilding at $time, dir $PWD"
make clean
make SQL_JAVASDK_ENABLE=ON
echo "rebuild succeed"
popd
# set this to crontab
# e.g. 0 0 * * * sh /Users/ark/Code/OpenMLDB/rebuild.sh >> /Users/ark/Code/OpenMLDB/rebuild.log 2>&1
#. will do it every day (at 00:00)

# test crontab
# if you use 'xx yy * * *', crontab will run it next day, only all * works(every minute), but it's dangerous.
# so make a crontab env:
# 1.crontab -e, then add '* * * * *  /usr/bin/env > /Users/ark/cron-env', cron-env will update every minute.
# 2.create a exec 'run-as-cron':
# ```
# !/bin/sh
# 
# . "$1"
# exec /usr/bin/env -i "$SHELL" -c ". $1; $2"
# ```
# 3. chmod +x, then run it. usage: run-as-cron <cron-environment> <command>
# 4. ./run-as-cron cron-env "sh xxx/rebuild.sh > xxx/rebuild.log 2>&1"

shell.sh

ps axu | grep xx | awk '{print $2}' | xargs kill -9
netstat -ltnup

watch-pr-ci.py

import re
import os
import subprocess
import sys
import time

# output = subprocess.check_output(['gh','-R','4paradigm/OpenMLDB', 'pr', 'checks', '3312']) # -R 4paradigm/OpenMLDB pr checks 3312'])
# print(output)

# result = subprocess.run(['gh','-R','4paradigm/OpenMLDB', 'pr', 'checks', '3312'])#, shell=True)

from subprocess import Popen, PIPE

repo = "4paradigm/OpenMLDB"

assert len(sys.argv) >= 2, "input pr is needed"

pr_list = sys.argv[1]
if len(sys.argv) > 2:
    repo = sys.argv[2]
if not pr_list:
    print("no pr input. exit.")
    exit(-1)

prs = []
if pr_list == 'me':
    p = Popen(f"gh pr list -A vagetablechicken -R {repo}".split(" "), stdin=PIPE, stdout=PIPE, stderr=PIPE)
    (
        output,
        err,
    ) = p.communicate()  # b"input data that is passed to subprocess' stdin")
    rc = p.returncode
    # print(output)

    # split jobs result
    result = output.decode("utf-8")
    prs = [pr.split("\t")[0] for pr in result.split("\n") if pr]
else:
    # pr -> prs, split by ,
    prs = pr_list.split(",")
watchs = [
    "cpp",
    "macos-cpp",
    "coverage",
    "python-sdk",
    "java-sdk",
    "openmldb-test-python",
    "go-sdk",
    "openmldb-tool-test",
]
status = []


def check_pr(pr):
    # print(f'check {pr} in repo {repo}')
    pr_status = []

    def get_job_status():
        p = Popen(
            ["gh", "-R", repo, "pr", "checks", pr], stdin=PIPE, stdout=PIPE, stderr=PIPE
        )
        (
            output,
            err,
        ) = p.communicate()  # b"input data that is passed to subprocess' stdin")
        rc = p.returncode
        # print(output)

        # split jobs result
        result = output.decode("utf-8")
        jobs = [job.split("\t") for job in result.split("\n")]
        # print(jobs)
        return jobs

    jobs = get_job_status()
    # add a fail to test
    # jobs.append(['cpp', 'fail', '', 'https://github.com/4paradigm/OpenMLDB/actions/runs/5220800424/jobs/9425957322'])

    for job in jobs:
        if job[0] not in watchs:
            continue

        # print('check', job)
        if job[1] == "fail":
            # parse run id
            x = re.search("runs/[0-9]+", job[3])
            if not x:
                print("no run id?", job[3])
            else:
                id = x.group(0)
                id = id[5:]
                print("rerun", id)
                try:
                    subprocess.check_output(
                        f"gh -R 4paradigm/OpenMLDB run rerun --failed {id}", shell=True
                    )
                except subprocess.CalledProcessError as e:
                    print(e.output)
        pr_status.append((job[0], job[1]))

    all_pass = False
    if all(s == "pass" for _, s in pr_status):
        print(f"pr {pr} all pass")
        all_pass = True
    else:
        print(f"pr {pr} not all pass:", pr_status)
    return all_pass, pr_status


print(f"start watch {prs} in repo {repo}, watchs {watchs}")

# pr done if all pass 3 times
pr_st = { pr: 0 for pr in prs }
while True:
    for pr, succ in pr_st.items():
        if succ > 2:
            continue
        done, _ = check_pr(pr)
        if done:
            pr_st[pr] += 1
    if all(succ > 2 for succ in pr_st.values()):
        print("all pr pass 3 times. exit.")
        exit(0)
    print(pr_st)
    time.sleep(5 * 60)

assert False, "?"

rebuild.sh needs path env? new make will get error cmake: No such file or directory
solution: export PATH(if it contains cmake bin path, skip add) before make
PATH exists but not export by default, so we should export it. So the root Makefile can use $PATH, you can echo $PATH in Makefile to test.

gh run list -w cicd and other workflow, watch all? python is better