commit a0e39f9c7adc120921fab28da6d88a4857a70547
parent f6667bb506735310994c5b90eb9abec49f03a9e1
Author: Hiltjo Posthuma <hiltjo@codemadness.org>
Date: Sun, 9 Feb 2025 12:07:00 +0100
README, sfeedrc.5: improve If-Modified-Since example
Improvements:
- Use the exact server timestamp to compare modified content (curl -R option).
Using the client timestamp is imprecise.
- Do not use the modified timestamp of the ETag for If-Modified-Since this can
differ as well.
Probably too obvious to mention but this of course assumes the filesystem can be used
to store the modification timestamp for the modified content.
Based on feedback from free_electrician, thanks!
Diffstat:
| M | README | | | 30 | ++++++++++++++++++++++++------ |
| M | sfeedrc.5 | | | 28 | +++++++++++++++++++++------- |
2 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/README b/README
@@ -644,9 +644,9 @@ Caching, incremental data updates and bandwidth saving
For servers that support it some incremental updates and bandwidth saving can
be done by using the "ETag" HTTP header.
-Create a directory for storing the ETags per feed:
+Create a directory for storing the ETags and modification timestamps per feed:
- mkdir -p ~/.sfeed/etags/
+ mkdir -p ~/.sfeed/etags ~/.sfeed/lastmod
The curl ETag options (--etag-save and --etag-compare) can be used to store and
send the previous ETag header value. curl version 7.73+ is recommended for it
@@ -665,13 +665,31 @@ file:
# fetch(name, url, feedfile)
fetch() {
- etag="$HOME/.sfeed/etags/$(basename "$3")"
+ basename="$(basename "$3")"
+ etag="$HOME/.sfeed/etags/${basename}"
+ lastmod="$HOME/.sfeed/lastmod/${basename}"
+ output="${sfeedtmpdir}/feeds/${filename}.xml"
+
curl \
- -L --max-redirs 0 -H "User-Agent:" -f -s -m 15 \
+ -f -s -m 15 \
+ -L --max-redirs 0 \
+ -H "User-Agent: sfeed" \
--compressed \
--etag-save "${etag}" --etag-compare "${etag}" \
- -z "${etag}" \
- "$2" 2>/dev/null
+ -R -o "${output}" \
+ -z "${lastmod}" \
+ "$2" 2>/dev/null || return 1
+
+ # succesful, but no file written: assume it is OK and Not Modified.
+ [ -e "${output}" ] || return 0
+
+ # use server timestamp from curl -R to set Last-Modified.
+ touch -r "${output}" "${lastmod}" 2>/dev/null
+ cat "${output}" 2>/dev/null
+ # use write output status, other errors are ignored here.
+ fetchstatus="$?"
+ rm -f "${output}" 2>/dev/null
+ return "${fetchstatus}"
}
These options can come at a cost of some privacy, because it exposes
diff --git a/sfeedrc.5 b/sfeedrc.5
@@ -1,4 +1,4 @@
-.Dd November 20, 2024
+.Dd February 9, 2025
.Dt SFEEDRC 5
.Os
.Sh NAME
@@ -217,21 +217,35 @@ HTTP clients based on the User-Agent request header.
.Pp
Example:
.Bd -literal -offset 4n
-etagpath="$HOME/.sfeed/etags"
-mkdir -p "${etagpath}"
+mkdir -p "$HOME/.sfeed/etags" "$HOME/.sfeed/lastmod"
# fetch(name, url, feedfile)
fetch() {
- etag="${etagpath}/$(basename "$3")"
+ basename="$(basename "$3")"
+ etag="$HOME/.sfeed/etags/${basename}"
+ lastmod="$HOME/.sfeed/lastmod/${basename}"
+ output="${sfeedtmpdir}/feeds/${filename}.xml"
curl \e
+ -f -s -m 15 \e
-L --max-redirs 0 \e
-H "User-Agent: sfeed" \e
- -f -s -m 15 \e
--compressed \e
--etag-save "${etag}" --etag-compare "${etag}" \e
- -z "${etag}" \e
- "$2" 2>/dev/null
+ -R -o "${output}" \e
+ -z "${lastmod}" \e
+ "$2" 2>/dev/null || return 1
+
+ # succesful, but no file written: assume it is OK and Not Modified.
+ [ -e "${output}" ] || return 0
+
+ # use server timestamp from curl -R to set Last-Modified.
+ touch -r "${output}" "${lastmod}" 2>/dev/null
+ cat "${output}" 2>/dev/null
+ # use write output status, other errors are ignored here.
+ fetchstatus="$?"
+ rm -f "${output}" 2>/dev/null
+ return "${fetchstatus}"
}
.Ed
.Pp
