find-optional-output-hubs.sh

#!/usr/bin/env bash
# 
# This will search GitHub for hubs that use quantiles, CDF, or PMF output
# types and check if they use "optional" in their hub-config/tasks.json
# files.
echo "======== SEARCHING GITHUB ============"
curl -L \
  -H "Accept: application/vnd.github+json" \
  -H "Authorization: Bearer $(gh auth token)" \
  -H "X-GitHub-Api-Version: 2022-11-28" \
  "https://api.github.com/search/code?q=path:hub-config+quantile+language:json" \
  | jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s > quantile.json
curl -L \
  -H "Accept: application/vnd.github+json" \
  -H "Authorization: Bearer $(gh auth token)" \
  -H "X-GitHub-Api-Version: 2022-11-28" \
  "https://api.github.com/search/code?q=path:hub-config/+pmf+language:json" \
| jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s > pmf.json
curl -L \
  -H "Accept: application/vnd.github+json" \
  -H "Authorization: Bearer $(gh auth token)" \
  -H "X-GitHub-Api-Version: 2022-11-28" \
  "https://api.github.com/search/code?q=path:hub-config/+cdf+language:json" \
| jq ".items[] | select(.name == \"tasks.json\") | {path: .url, repo: .repository.full_name}" \
| jq -s> cdf.json


echo "======== DOWNLOADING FILES ============"
echo ">>> QUANTILES"
thing=quantile
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length quantile.json) - 1)));
do
  repo=$(jq -r ".["$i"].repo" quantile.json)
  path=$(jq -r ".["$i"].path" quantile.json)
  if [[ -e "$repo" ]]; then
    # echo "--- already downloaded: $repo"
    :
  else
    echo "Downloading $repo ==================="
    mkdir -p "$repo"
    (
      cd "$repo"
      curl -L \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: Bearer $(gh auth token)" \
        -H "X-GitHub-Api-Version: 2022-11-28" \
      "$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
    )
  fi
  n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
  if [[ $n == 0 ]]; then
    :
  else
    echo "| $n | https://github.com/$repo |"
  fi
done
echo ">>> CDF" 
thing=cdf
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length cdf.json) - 1)));
do
  repo=$(jq -r ".["$i"].repo" cdf.json)
  path=$(jq -r ".["$i"].path" cdf.json)
  if [[ -e "$repo" ]]; then
    # echo "--- already downloaded: $repo"
    :
  else
    echo "Downloading $repo ==================="
    mkdir -p "$repo"
    (
      cd "$repo"
      curl -L \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: Bearer $(gh auth token)" \
        -H "X-GitHub-Api-Version: 2022-11-28" \
      "$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
    )
  fi
  n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
  if [[ $n == 0 ]]; then
    :
  else
    echo "| $n | https://github.com/$repo |"
  fi
done
echo ">>> PMF" 
thing=pmf
echo "| fields with optional ${thing} | repository |"
echo "| ----------------------------- | ---------- |"
for i in $(seq 0 $(( $(jq length pmf.json) - 1)));
do
  repo=$(jq -r ".["$i"].repo" pmf.json)
  path=$(jq -r ".["$i"].path" pmf.json)
  if [[ -e "$repo" ]]; then
    # echo "--- already downloaded: $repo"
    :
  else
    echo "Downloading $repo ==================="
    mkdir -p "$repo"
    (
      cd "$repo"
      curl -L \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: Bearer $(gh auth token)" \
        -H "X-GitHub-Api-Version: 2022-11-28" \
      "$path" | jq -r .content | base64 --decode > tasks.json || echo "oops: $repo"
    )
  fi
  
  n=$(jq "[.rounds[].model_tasks[].output_type.${thing}.output_type_id.optional | if . == null then 0 else 1 end] | add" "$repo/tasks.json")
  if [[ $n == 0 ]]; then
    :
  else
    echo "| $n | https://github.com/$repo |"
  fi
done

as of 2024-09-18:

$ bash find-optional-output-hubs.sh 
======== SEARCHING GITHUB ============
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  159k  100  159k    0     0   306k      0 --:--:-- --:--:-- --:--:--  306k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 23947  100 23947    0     0   102k      0 --:--:-- --:--:-- --:--:--  102k
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 42538  100 42538    0     0   155k      0 --:--:-- --:--:-- --:--:--  155k
======== DOWNLOADING FILES ============
>>> QUANTILES

fields with optional quantile	repository
1	https://github.com/midas-network/covid19-scenario-modeling-hub
4	https://github.com/Testing-Forecast-Actions/ScenarioModellingHub
2	https://github.com/micokoch/simple-hub
13	https://github.com/midas-network/example_round-scenariohub
3	https://github.com/european-modelling-hubs/RespiCompass
2	https://github.com/midas-network/rsv-scenario-modeling-hub
6	https://github.com/midas-network/flu-scenario-modeling-hub
1	https://github.com/Testing-Forecast-Actions/TestingValidations
1	https://github.com/hubverse-org/example-complex-scenario-hub
4	https://github.com/midas-network/covid19-smh-research
2	https://github.com/LucieContamin/example_smh

>>> CDF

fields with optional cdf	repository
2	https://github.com/hubverse-org/flusight_hub_archive

>>> PMF

fields with optional pmf	repository
1	https://github.com/hubverse-org/flusight_hub_archive