Changeset 1210 for dass-tools/usr

Timestamp:

Apr 12, 2016, 4:58:03 PM (9 years ago)

Author:

joergs

Message:

make mirror script more flexible.

Use configuration files and helper scripts.

Location:

dass-tools/usr/bin

Files:

• mirror-adapt-repository.sh (added)
• mirror-obs.sh (modified) (1 diff)
• mirror-urls.sh (modified) (1 diff)

dass-tools/usr/bin/mirror-obs.sh

@@ -1 +1 @@
 #!/bin/bash
 
-SOURCE_OPENSUSE="ftp5.gwdg.de::pub/opensuse/"
+set -o errexit -o nounset
 
-REPOS="repositories/home:/dassit
-"
+CONFIG=${1:-/etc/mirror-obs.conf}
 
-#
-# disabled repos:
-#
-#distribution/11.2/repo update/11.2 distribution/12.1/repo update/12.1
+if ! [ -r "$CONFIG" ]; then
+  echo "failed to read config file $CONFIG."
+  exit 1
+fi
+source $CONFIG
 
 RSYNC_DELETE="--delete"
-
-#RSYNC_OPTS="-v -rlLptx --safe-links --stats \
-#        --exclude=*.src.rpm --exclude=*.nosrc.rpm --exclude=*.spm \
-#        --exclude=*.patch.rpm"
-
-RSYNC_OPTS="-v -rlLptx --safe-links --stats"
-
-DEST="/space/installtree/install/download.opensuse.org"
+RSYNC_OPTS="-v -rlLptx --safe-links --stats $RSYNC_EXTRA_OPTS"
 
 rsync_if_exists()
 {
-        src=$1
-        dest=$2
+    src=$1
+    dest=$2
 
-        rsync -q $src || return 1
-        echo rsync-quelle $src gefunden.
-        [ -d "$dest" ] || return 2
-        rsync $RSYNC_OPTS $RSYNC_DELETE $src ./$dest
+    rsync -q $src || return 1
+    echo "found rsync source: $src"
+    [ -d "$dest" ] || return 2
+    rsync $RSYNC_OPTS $RSYNC_DELETE $src $dest
 }
 
 # ---------------- mirror openSUSE repositories ------------------------
-FAILED_REPOS=""
-SUCCESSFUL=""
+FAILED=""
+SUCCESS=""
+
 
 cd $DEST
 
-for DIR in $REPOS 
-do
+for DIR in $REPOS; do
   SOURCE_DIR=${SOURCE_OPENSUSE}/${DIR}/.
   DEST_DIR=${DEST}/${DIR}/.
   mkdir -p ${DEST_DIR}
-  rsync_if_exists  "${SOURCE_OPENSUSE}$DIR/" "$DIR"
-  case $? in
-        1)      echo Distribution $DIR nicht gefunden
-                FAILED_REPOS="$FAILED_REPOS $DIR"
-                ;;
-        2)      echo Zielverzeichnis $(pwd)/$DIR nicht vorhanden
-                FAILED_REPOS="$FAILED_REPOS $DIR"
-                ;;
-        0)
-                SUCCESSFUL="$SUCCESSFUL $DIR"
-                ;;
-        *)
-                echo Fehler $?
-                FAILED_REPOS="$FAILED_REPOS $DIR"
-                ;;
+  rc=0
+  rsync_if_exists  "${SOURCE_DIR}" "${DEST_DIR}" || rc=$?
+  case $rc in
+    1)  echo Distribution $DIR nicht gefunden
+        FAILED="$FAILED $DIR"
+        ;;
+    2)  echo Zielverzeichnis ${DEST_DIR} nicht vorhanden
+        FAILED="$FAILED $DIR"
+        ;;
+    0)
+        SUCCESS="$SUCCESS $DIR"
+        if [ "${POST_SCRIPT:-}" ]; then
+            $POST_SCRIPT ${DEST_DIR}
+        fi
+        ;;
+    *)
+        echo Fehler $?
+        FAILED="$FAILED $DIR"
+        ;;
   esac
+  echo
+  echo
 done
 
-# /usr/local/sbin/mirror-report-errors.sh mirror-osb "$SUCCESSFUL" "$FAILED_REPOS"
-# 
-# # adapt repository sources
-# for i in `find $DEST/repositories -name "*.repo" -a ! -name "vermkv-*.repo"`; do
-#       basename=`basename $i`
-#       dirname=`dirname $i`
-#         sed "s|http://|http://install.vermkv/freigabe/|" $i > $dirname/vermkv-$basename 
-# done
-
+if [ "${REPORT_SCRIPT:-}" ]; then
+    $REPORT_SCRIPT mirror-urls "$SUCCESS" "$FAILED"
+fi

dass-tools/usr/bin/mirror-urls.sh

@@ -1 +1 @@
 #!/bin/bash
 
-export http_proxy=http://proxy:3128
-export ftp_proxy=http://proxy:3128
+set -o errexit -o nounset
 
-NSCA_CMD="/usr/local/bin/nagios_send_nsca.sh"
+CONFIG=${1:-/etc/mirror-urls.conf} 
 
-PROJECTS="
-http://download.videolan.org/pub/vlc/SuSE/11.2/
-http://download.videolan.org/pub/vlc/SuSE/12.1/
-"
+# default value, may be overwritten by CONFIG
+DEST="/cds/linux/mirror/"
 
-# TODO:
-# not working, because dir listings are forbidden
-# http://download.nvidia.com/opensuse/11.2/
-# (with hftp://download.nvidia.com/opensuse/11.2/ directory listings are possible,
-#  but file access in denied)
+if ! [ -r "$CONFIG" ]; then
+  echo "failed to read config file $CONFIG."
+  exit 1
+fi
+source $CONFIG
 
-SUCCESSFUL=""
 FAILED=""
-
-DEST="/anlagen/vmlxlamp/repo/"
+SUCCESS=""
 
 cd $DEST
 
-
 for url in $PROJECTS; do
-  echo $url
-  protocol=`sed -r "s|^([a-zA-Z]+)://.*|\1|" <<< $url`
-  host=`sed -r "s|^$protocol://([^/]+)/.*|\1|" <<< $url`
-  path=`sed -r "s|$protocol://$host||" <<< $url`
-  mkdir -p $host/$path
-  set -x
-  if ! lftp -c "open -e \"mirror --verbose=1 --continue --delete $path/. $DEST/$host/$path\" ${protocol}://${host}"; then
-        FAILED="$FAILED $url"
-  else
-        SUCCESS="$SUCCESS $url"
-  fi
-  set +x
+    printf "$url:\n"
+    protocol=`sed -r "s|^([a-zA-Z]+)://.*|\1|" <<< $url`
+    host=`sed -r "s|^$protocol://([^/]+)/.*|\1|" <<< $url`
+    path=`sed -r "s|$protocol://$host||" <<< $url`
+    mkdir -p $host/$path
+    DESTDIR=$DEST/$host/$path
+    #set -x
+    if ! lftp -c "open -e \"mirror --verbose=1 --parallel=20 --continue --delete $path/. ${DESTDIR}\" ${protocol}://${host}"; then
+        set +x
+        printf "    failed\n"
+        FAILED="$FAILED $url"
+    else
+        set +x
+        if [ "${POST_SCRIPT:-}" ]; then
+            $POST_SCRIPT ${DESTDIR}
+        fi
+        printf "    ${DESTDIR}\n"
+        SUCCESS="$SUCCESS $url"
+    fi
+    printf "\n"
 done
 
-# adapt repository sources
-for i in `find $DEST -name "*.repo" -a ! -name "vermkv-*.repo"`; do
-        basename=`basename $i`
-        dirname=`dirname $i`
-        sed "s|http://|http://install.vermkv/freigabe/|" $i > $dirname/vermkv-$basename 
-done
-
-/usr/local/sbin/mirror-report-errors.sh mirror-urls "$SUCCESS" "$FAILED" 
+if [ "${REPORT_SCRIPT:-}" ]; then
+    $REPORT_SCRIPT mirror-urls "$SUCCESS" "$FAILED"
+fi