initHEADmain

1 files changed, 19 insertions, 0 deletions

diff --git a/crawl.sh b/crawl.sh
new file mode 100755
index 0000000..dee87c6
--- /dev/null
+++ b/crawl.sh
@@ -0,0 +1,19 @@
+set -e; shopt -s globstar nullglob
+while read -r name offset base; do
+  [[ "$base" ]] && wget2 "$base" \
+    --mirror \
+    --page-requisites \
+    --no-host-directories \
+    --directory-prefix="$name" \
+    --cut-dirs=$offset \
+    --cut-url-get-vars \
+    --cut-file-get-vars \
+    --no-check-certificate \
+    --local-encoding=utf-8 \
+    "$@" \
+    | tee "$name.crawl.$(date -uIs).log"
+
+  for f in **/*.orig; do
+    mv "$f" "${f%%.orig}"
+  done
+done