#!/usr/bin/env bash
# based on https://github.com/kajeagentspi/Datahoarder
set -e

URL=$1
ROOT_PATH=$2
LIST=./list.txt
MAX_CONNECTIONS_PER_SERVER=16
USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"

usage() {
	cat <<EOF
Uses wget's spider with aria2c's parallel for downloading open
directories.
Usage: $SCRIPT_NAME [options] URL PATH
EOF
}

spider() {
	local logfile=./opendir-$$.log
	wget -o $logfile -e robots=off -r --no-parent --spider -U "$USER_AGENT" "$URL" || true
	#Grabs all lines with the pattern --2017-07-12 15:40:31-- then from the results removes everthing that ends in / (meaning it's a directory
	#then removes pattern from every line
	grep -B 2 -E '... 404 Not Found|... 403 Forbidden|... 301 Moved Permanently' $logfile | \
	grep -i '^--[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]--' | \
	grep '[^'/']$'  | sed -e 's/^--[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]--  //g' > $logfile.tmp
	while read line; do
		sed -i "\|$line|d" $logfile
	done < $logfile.tmp
	cat $logfile | grep -i '^--[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]--' | \
	grep '[^'/']$'  | sed -e 's/^--[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]--  //g' > $LIST
	#Delete the folder made by wget (deletes all empty directories in the directory this script is run
	find . -type d -empty -delete #If you have a fix for this contact me since it should only delete the folder created by wget
}

if [[ -z $1 || -z $2 || $# -ge 3 ]]; then
	usage
	exit 1
fi

echo "Creating list of urls..."
spider
echo "List created!"