rdo mirroring

centos7.mirror.sh

#!/bin/bash

#rsync  -avSHP --delete --exclude "local*" --exclude "isos" --exclude "atomic" rsync://mirrors.serenitytek.com/centos/7/ /home/aschultz/mirror/CentOS/7/

MIRROR_LIST=(rsync://mirrors.syringanetworks.net/centos/7/ rsync://mirrors.serenitytek.com/centos/7/ rsync://bay.uchicago.edu/CentOS/7/ rsync://mirror.oss.ou.edu/centos/7/ rsync://mirrors.xmission.com/centos/7/ rsync://mirror.chpc.utah.edu/pub/centos/7/)
MIRROR_DIR=/var/lib/www/mirror/repos/centos/7/
MIRROR_EXCLUDES="local* atomic configmanagement cr dotnet fasttrack isos nfv rt sclo"

mkdir -p $MIRROR_DIR

EXCLUDES=""
for E in $MIRROR_EXCLUDES; do
   echo "Excluding $E"
   EXCLUDES="$EXCLUDES --exclude $E"
done

echo "Picking mirror...."
size=${#MIRROR_LIST[@]}
index=$(($RANDOM % $size))
MIRROR_RSYNC=${MIRROR_LIST[$index]}
echo "Running rsync from $MIRROR_RSYNC to $MIRROR_DIR"
rsync -avSHP --delete $EXCLUDES $MIRROR_RSYNC $MIRROR_DIR

centos8.mirror.sh

#!/bin/bash

#rsync  -avSHP --delete --exclude "local*" --exclude "isos" --exclude "atomic" rsync://mirrors.serenitytek.com/centos/8/ /home/aschultz/mirror/CentOS/8/

MIRROR_LIST=(rsync://mirrors.syringanetworks.net/centos/8/ rsync://mirrors.serenitytek.com/centos/8/ rsync://bay.uchicago.edu/CentOS/8/ rsync://mirror.oss.ou.edu/centos/8/ rsync://mirrors.xmission.com/centos/8/ rsync://mirror.chpc.utah.edu/pub/centos/8/)
MIRROR_DIR=/var/lib/www/mirror/repos/centos/8/
MIRROR_EXCLUDES="local* centosplus fasttrack isos *aarch64* *ppc64le*"

mkdir -p $MIRROR_DIR

EXCLUDES=""
for E in $MIRROR_EXCLUDES; do
   echo "Excluding $E"
   EXCLUDES="$EXCLUDES --exclude $E"
done

echo "Picking mirror...."
size=${#MIRROR_LIST[@]}
index=$(($RANDOM % $size))
MIRROR_RSYNC=${MIRROR_LIST[$index]}
echo "Running rsync from $MIRROR_RSYNC to $MIRROR_DIR"
rsync -avSHP --delete $EXCLUDES $MIRROR_RSYNC $MIRROR_DIR

nginx.mirror.conf

# mkdir -p /var/lib/www/{mirror,cache}
# chown nginx: -R /var/lib/www
# For selinux:
# setsebool -P httpd_can_network_relay 1
# semanage fcontext -a -t httpd_sys_rw_content_t "/var/lib/www(/.*)?

proxy_cache_path /var/lib/www/cache levels=1:2 keys_zone=reverse:10m max_size=1g inactive=24h use_temp_path=off;
proxy_cache_key "$scheme$request_method$host$request_uri";
 
server {
    listen 80;
    root /var/lib/www/mirror;
    location / { autoindex on; }
    location /rdo/ {
        proxy_pass https://trunk.rdoproject.org/;
        #proxy_set_header Host $host;
        proxy_buffering on;
        proxy_cache reverse;
        proxy_cache_valid 200 1d;
        proxy_cache_use_stale  error timeout invalid_header updating
                               http_500 http_502 http_503 http_504;
        add_header X-Proxy-Cache $upstream_cache_status;
    }
}