crawl.sh


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

set -e; shopt -s globstar nullglob
while read -r name offset base; do
  [[ "$base" ]] && wget2 "$base" \
    --mirror \
    --page-requisites \
    --no-host-directories \
    --directory-prefix="$name" \
    --cut-dirs=$offset \
    --cut-url-get-vars \
    --cut-file-get-vars \
    --no-check-certificate \
    --local-encoding=utf-8 \
    "$@" \
    | tee "$name.crawl.$(date -uIs).log"

  for f in **/*.orig; do
    mv "$f" "${f%%.orig}"
  done
done