From ddbbfedf15e970c02e128294890adbeb97c4a5ae Mon Sep 17 00:00:00 2001
From: uakci <uakci@uakci.pl>
Date: Tue, 15 Mar 2022 19:13:44 +0100
Subject: init

---
 crawl.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100755 crawl.sh

(limited to 'crawl.sh')

diff --git a/crawl.sh b/crawl.sh
new file mode 100755
index 0000000..dee87c6
--- /dev/null
+++ b/crawl.sh
@@ -0,0 +1,19 @@
+set -e; shopt -s globstar nullglob
+while read -r name offset base; do
+  [[ "$base" ]] && wget2 "$base" \
+    --mirror \
+    --page-requisites \
+    --no-host-directories \
+    --directory-prefix="$name" \
+    --cut-dirs=$offset \
+    --cut-url-get-vars \
+    --cut-file-get-vars \
+    --no-check-certificate \
+    --local-encoding=utf-8 \
+    "$@" \
+    | tee "$name.crawl.$(date -uIs).log"
+
+  for f in **/*.orig; do
+    mv "$f" "${f%%.orig}"
+  done
+done
-- 
cgit v1.2.3