sfeed

sfeed_update (raw) (5664B)

---

 #!/bin/sh
 # update feeds, merge with old feeds.
 # NOTE: assumes "sfeed_*" executables are in $PATH.
 
 # defaults
 sfeedpath="$HOME/.sfeed/feeds"
 
 # used for processing feeds concurrently: wait until ${maxjobs} amount of
 # feeds are finished at a time.
 maxjobs=8
 
 # load config (evaluate shellscript).
 # loadconfig(configfile)
 loadconfig() {
 	# allow to specify config via argv[1].
 	if [ "$1" != "" ]; then
 		# get absolute path of config file required for including.
 		config="$1"
 		path=$(readlink -f "${config}" 2>/dev/null)
 	else
 		# default config location.
 		config="$HOME/.sfeed/sfeedrc"
 		path="${config}"
 	fi
 
 	# config is loaded here to be able to override $sfeedpath or functions.
 	if [ -r "${path}" ]; then
 		. "${path}"
 	else
 		printf "Configuration file \"%s\" cannot be read.\n" "${config}" >&2
 		echo "See the sfeedrc.example file or the sfeedrc(5) man page for an example." >&2
 		exit 1
 	fi
 }
 
 # log(name, s)
 log() {
 	printf '[%s] %-50.50s %s\n' "$(date +'%H:%M:%S')" "$1" "$2"
 }
 
 # log_error(name, s)
 log_error() {
 	printf '[%s] %-50.50s %s\n' "$(date +'%H:%M:%S')" "$1" "$2" >&2
 	# set error exit status indicator for parallel jobs.
 	rm -f "${sfeedtmpdir}/ok"
 }
 
 # fetch a feed via HTTP/HTTPS etc.
 # fetch(name, url, feedfile)
 fetch() {
 	# fail on redirects, hide User-Agent, timeout is 15 seconds.
 	curl -L --max-redirs 0 -H "User-Agent:" -f -s -m 15 \
 		"$2" 2>/dev/null
 }
 
 # convert encoding from one encoding to another.
 # convertencoding(name, from, to)
 convertencoding() {
 	if [ "$2" != "" ] && [ "$3" != "" ] && [ "$2" != "$3" ]; then
 		iconv -cs -f "$2" -t "$3" 2> /dev/null
 	else
 		# else no convert, just output.
 		cat
 	fi
 }
 
 # parse and convert input, by default XML to the sfeed(5) TSV format.
 # parse(name, feedurl, basesiteurl)
 parse() {
 	sfeed "$3"
 }
 
 # filter fields.
 # filter(name)
 filter() {
 	cat
 }
 
 # merge raw files: unique sort by id, title, link.
 # merge(name, oldfile, newfile)
 merge() {
 	sort -t '	' -u -k6,6 -k2,2 -k3,3 "$2" "$3" 2>/dev/null
 }
 
 # order by timestamp (descending).
 # order(name)
 order() {
 	sort -t '	' -k1rn,1
 }
 
 # internal handler to fetch and process a feed.
 # _feed(name, feedurl, [basesiteurl], [encoding])
 _feed() {
 	name="$1"
 	feedurl="$2"
 	basesiteurl="$3"
 	encoding="$4"
 
 	filename="$(printf '%s' "${name}" | tr '/' '_')"
 	sfeedfile="${sfeedpath}/${filename}"
 	tmpfeedfile="${sfeedtmpdir}/feeds/${filename}"
 
 	# if file does not exist yet create it.
 	[ -e "${sfeedfile}" ] || touch "${sfeedfile}" 2>/dev/null
 
 	if ! fetch "${name}" "${feedurl}" "${sfeedfile}" > "${tmpfeedfile}.fetch"; then
 		log_error "${name}" "FAIL (FETCH)"
 		return 1
 	fi
 
 	# try to detect encoding (if not specified). if detecting the encoding fails assume utf-8.
 	[ "${encoding}" = "" ] && encoding=$(sfeed_xmlenc < "${tmpfeedfile}.fetch")
 
 	if ! convertencoding "${name}" "${encoding}" "utf-8" < "${tmpfeedfile}.fetch" > "${tmpfeedfile}.utf8"; then
 		log_error "${name}" "FAIL (ENCODING)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.fetch"
 
 	# if baseurl is empty then use feedurl.
 	if ! parse "${name}" "${feedurl}" "${basesiteurl:-${feedurl}}" < "${tmpfeedfile}.utf8" > "${tmpfeedfile}.tsv"; then
 		log_error "${name}" "FAIL (PARSE)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.utf8"
 
 	if ! filter "${name}" < "${tmpfeedfile}.tsv" > "${tmpfeedfile}.filter"; then
 		log_error "${name}" "FAIL (FILTER)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.tsv"
 
 	# new feed data is empty: no need for below stages.
 	if [ ! -s "${tmpfeedfile}.filter" ]; then
 		log "${name}" "OK"
 		return 0
 	fi
 
 	if ! merge "${name}" "${sfeedfile}" "${tmpfeedfile}.filter" > "${tmpfeedfile}.merge"; then
 		log_error "${name}" "FAIL (MERGE)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.filter"
 
 	if ! order "${name}" < "${tmpfeedfile}.merge" > "${tmpfeedfile}.order"; then
 		log_error "${name}" "FAIL (ORDER)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.merge"
 
 	# copy
 	if ! cp "${tmpfeedfile}.order" "${sfeedfile}"; then
 		log_error "${name}" "FAIL (COPY)"
 		return 1
 	fi
 	rm -f "${tmpfeedfile}.order"
 
 	# OK
 	log "${name}" "OK"
 	return 0
 }
 
 # fetch and process a feed in parallel.
 # feed(name, feedurl, [basesiteurl], [encoding])
 feed() {
 	# wait until ${maxjobs} are finished: will stall the queue if an item
 	# is slow, but it is portable.
 	[ ${signo} -ne 0 ] && return
 	[ $((curjobs % maxjobs)) -eq 0 ] && wait
 	[ ${signo} -ne 0 ] && return
 	curjobs=$((curjobs + 1))
 
 	_feed "$@" &
 }
 
 cleanup() {
 	# remove temporary directory with feed files.
 	rm -rf "${sfeedtmpdir}"
 }
 
 sighandler() {
 	signo="$1"
 	# ignore TERM signal for myself.
 	trap -- "" TERM
 	# kill all running children >:D
 	kill -TERM -$$
 }
 
 feeds() {
 	printf "Configuration file \"%s\" is invalid or does not contain a \"feeds\" function.\n" "${config}" >&2
 	echo "See sfeedrc.example for an example." >&2
 }
 
 main() {
 	# job counter.
 	curjobs=0
 	# signal number received for parent.
 	signo=0
 	# SIGINT: signal to interrupt parent.
 	trap -- "sighandler 2" "INT"
 	# SIGTERM: signal to terminate parent.
 	trap -- "sighandler 15" "TERM"
 	# load config file.
 	loadconfig "$1"
 	# fetch feeds and store in temporary directory.
 	sfeedtmpdir="$(mktemp -d '/tmp/sfeed_XXXXXX')"
 	mkdir -p "${sfeedtmpdir}/feeds"
 	touch "${sfeedtmpdir}/ok"
 	# make sure path exists.
 	mkdir -p "${sfeedpath}"
 	# fetch feeds specified in config file.
 	feeds
 	# wait till all feeds are fetched (concurrently).
 	[ ${signo} -eq 0 ] && wait
 	# check error exit status indicator for parallel jobs.
 	[ -f "${sfeedtmpdir}/ok" ]
 	status=$?
 	# cleanup temporary files etc.
 	cleanup
 	# on signal SIGINT and SIGTERM exit with signal number + 128.
 	[ ${signo} -ne 0 ] && exit $((signo+128))
 	exit ${status}
 }
 
 [ "${SFEED_UPDATE_INCLUDE}" = "1" ] || main "$@"