commit 519b1d7ef4d2f2f5ccf2e00c56006e666382a9da
parent 36e2d18f125137e15ab3a2261f9f2ac4d468b8ba
Author: gearsix <gearsix@tuta.io>
Date: Mon, 14 Mar 2022 23:26:49 +0000
tidyup: moved a bunch of code from page.go -> content.go
Diffstat:
| A | content.go | | | 270 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | content_test.go | | | 52 | ++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| M | page.go | | | 302 | ++++++------------------------------------------------------------------------- |
| M | page_test.go | | | 46 | ---------------------------------------------- |
| M | pagr.go | | | 2 | +- |
| M | sitemap_test.go | | | 2 | +- |
6 files changed, 347 insertions(+), 327 deletions(-)
diff --git a/content.go b/content.go
@@ -0,0 +1,270 @@
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "io/fs"
+ "github.com/yuin/goldmark"
+ goldmarkext "github.com/yuin/goldmark/extension"
+ goldmarkparse "github.com/yuin/goldmark/parser"
+ goldmarkhtml "github.com/yuin/goldmark/renderer/html"
+ "notabug.org/gearsix/suti"
+ "path/filepath"
+ "os"
+ "strings"
+ "sort"
+ "time"
+)
+
+type Content string
+
+var contentExts = [6]string{
+ "", // pre-formatted text
+ ".txt", // plain-text
+ ".html", // HTML
+ ".md", // commonmark + extensions (linkify, auto-heading id, unsafe HTML)
+ ".gfm", // github-flavoured markdown
+ ".cm", // commonmark
+}
+
+func isContentExt(ext string) int {
+ for i, supported := range contentExts {
+ if ext == supported {
+ return i
+ }
+ }
+ return -1
+}
+
+func lastModFile(fpath string) (t time.Time) {
+ if fd, e := os.Stat(fpath); e != nil {
+ t = time.Now()
+ } else if !fd.IsDir() {
+ t = fd.ModTime()
+ } else { // find last modified file in directory (depth 1)
+ dir, err := os.ReadDir(fpath)
+ if err != nil {
+ return t
+ }
+
+ for i, d := range dir {
+ if fd, err := d.Info(); err == nil && (i == 0 || fd.ModTime().After(t)) {
+ t = fd.ModTime()
+ }
+ }
+ }
+ return
+}
+
+// LoadContentsDir parses all files/directories in `dir` into a `Content`.
+// For each directory, a new `Page` element will be generated, any file with a
+// filetype found in `contentExts`, will be parsed into a string of HTML
+// and appended to the `.Content` of the `Page` generated for it's parent
+// directory.
+func LoadContentsDir(dir string) (p []Page, e error) {
+ if _, e = os.Stat(dir); e != nil {
+ return
+ }
+ dir = filepath.Clean(dir)
+
+ pages := make(map[string]Page)
+ dmetas := make(map[string]Meta)
+
+ e = filepath.Walk(dir, func(fpath string, info fs.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if ignoreFile(fpath) {
+ return nil
+ }
+
+ if info.IsDir() {
+ path := pagePath(dir, fpath)
+ pages[path] = NewPage(path, lastModFile(fpath))
+ } else {
+ path := pagePath(dir, filepath.Dir(fpath))
+ page := pages[path]
+
+ if suti.IsSupportedDataLang(filepath.Ext(fpath)) > -1 {
+ var m Meta
+ if err = suti.LoadDataFilepath(fpath, &m); err == nil {
+ if strings.Contains(filepath.Base(fpath), "defaults.") ||
+ strings.Contains(filepath.Base(fpath), "default.") {
+ if meta, ok := dmetas[path]; ok {
+ m.MergeMeta(meta, false)
+ }
+ dmetas[path] = m
+ } else {
+ page.Meta.MergeMeta(m, true)
+ }
+ }
+ } else if isContentExt(filepath.Ext(fpath)) > -1 {
+ err = page.NewContentFromFile(fpath)
+ } else if suti.IsSupportedDataLang(filepath.Ext(fpath)) == -1 {
+ page.Assets = append(page.Assets, filepath.Join(path, filepath.Base(fpath)))
+ }
+
+ pages[path] = page
+ }
+ return err
+ })
+
+ for _, page := range pages {
+ page.applyDefaults(dmetas)
+ p = append(p, page)
+ }
+
+ sort.SliceStable(p, func(i, j int) bool {
+ if it, err := time.Parse(timefmt, p[i].Updated); err == nil {
+ if jt, err := time.Parse(timefmt, p[j].Updated); err == nil {
+ return it.After(jt)
+ }
+ }
+ return false
+ })
+
+ p = BuildSitemap(p)
+
+ return
+}
+
+// NewContentFromFile loads the file from `fpath` and converts it to HTML
+// from the language matching it's file extension (see below).
+// - ".txt" = plain-text
+// - ".md", ".gfm", ".cm" = various flavours of markdown
+// - ".html" = parsed as-is
+// Successful conversions are appended to `p.Contents`
+func NewContentFromFile(fpath string) (c Content, err error) {
+ var buf []byte
+ if f, err := os.Open(fpath); err == nil {
+ buf, err = io.ReadAll(f)
+ f.Close()
+ }
+ if err != nil {
+ return
+ }
+
+ var body string
+ for _, lang := range contentExts {
+ if filepath.Ext(fpath) == lang {
+ switch lang {
+ case "":
+ body = "<pre>" + string(buf) + "</pre>"
+ case ".txt":
+ body = convertTextToHTML(bytes.NewReader(buf))
+ case ".md":
+ fallthrough
+ case ".gfm":
+ fallthrough
+ case ".cm":
+ body, err = convertMarkdownToHTML(lang, buf)
+ case ".html":
+ body = string(buf)
+ default:
+ break
+ }
+ }
+ }
+ if len(body) == 0 {
+ err = fmt.Errorf("invalid filetype (%s) passed to NewContentFromFile",
+ filepath.Ext(fpath))
+ }
+ c = Content(body)
+ return
+}
+
+// convertTextToHTML parses textual data from `in` and line-by-line converts
+// it to HTML. Conversion rules are as follows:
+// - Blank lines (with escape characters trimmed) will close any opon tags
+// - If a text line is prefixed with a tab and no tag is open, it will open a <pre> tag
+// - Otherwise any line of text will open a <p> tag
+func convertTextToHTML(in io.Reader) (html string) {
+ var tag int
+ const p = 1
+ const pre = 2
+
+ fscan := bufio.NewScanner(in)
+ for fscan.Scan() {
+ line := fscan.Text()
+ if len(strings.TrimSpace(line)) == 0 {
+ switch tag {
+ case p:
+ html += "</p>\n"
+ case pre:
+ html += "</pre>\n"
+ }
+ tag = 0
+ } else if tag == 0 && line[0] == '\t' {
+ tag = pre
+ html += "<pre>" + line[1:] + "\n"
+ } else if tag == 0 || (tag == pre && line[0] != '\t') {
+ if tag == pre {
+ html += "</pre>\n"
+ }
+ tag = p
+ html += "<p>" + line
+ } else if tag == p {
+ html += " " + line
+ } else if tag == pre {
+ html += line[1:] + "\n"
+ }
+ }
+ if tag == p {
+ html += "</p>"
+ } else if tag == pre {
+ html += "</pre>"
+ }
+
+ return html
+}
+
+// convertMarkdownToHTML initialises a `goldmark.Markdown` based on `lang` and
+// returns values from calling it's `Convert` function on `in`.
+// Markdown `lang` options, see the code for specfics:
+// - ".gfm" = github-flavoured markdown
+// - ".cm" = standard commonmark
+// - ".md" (and anything else) = commonmark + extensions (linkify, auto-heading id, unsafe HTML)
+func convertMarkdownToHTML(lang string, buf []byte) (md string, err error) {
+ var markdown goldmark.Markdown
+ switch lang {
+ case ".gfm":
+ markdown = goldmark.New(
+ goldmark.WithExtensions(
+ goldmarkext.GFM,
+ goldmarkext.Table,
+ goldmarkext.Strikethrough,
+ goldmarkext.Linkify,
+ goldmarkext.TaskList,
+ ),
+ goldmark.WithParserOptions(
+ goldmarkparse.WithAutoHeadingID(),
+ ),
+ goldmark.WithRendererOptions(
+ goldmarkhtml.WithUnsafe(),
+ goldmarkhtml.WithHardWraps(),
+ ),
+ )
+ case ".cm":
+ markdown = goldmark.New()
+ case ".md":
+ fallthrough
+ default:
+ markdown = goldmark.New(
+ goldmark.WithExtensions(
+ goldmarkext.Linkify,
+ ),
+ goldmark.WithParserOptions(
+ goldmarkparse.WithAutoHeadingID(),
+ ),
+ goldmark.WithRendererOptions(
+ goldmarkhtml.WithUnsafe(),
+ ),
+ )
+ }
+
+ var out bytes.Buffer
+ err = markdown.Convert(buf, &out)
+ return out.String(), err
+}
diff --git a/content_test.go b/content_test.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+ "testing"
+ "os"
+)
+
+func TestLoadContentsDir(test *testing.T) {
+ var err error
+ tdir := test.TempDir()
+ if err = createTestContents(tdir); err != nil {
+ test.Errorf("failed to create test content: %s", err)
+ }
+
+ var p []Page
+ if p, err = LoadContentsDir(tdir); err != nil {
+ test.Fatalf("LoadContentsDir failed: %s", err)
+ }
+
+ validateTestPages(test, p, err)
+}
+
+func TestNewContentFromFile(test *testing.T) {
+ test.Parallel()
+
+ var err error
+ contents := map[string]string{
+ "txt": `test`,
+ "md": "**test**\ntest",
+ "gfm": "**test**\ntest",
+ "cm": "**test**",
+ "html": `<b>test</b>`,
+ }
+
+ tdir := test.TempDir()
+ contentsPath := func(ftype string) string {
+ return tdir + "/test." + ftype
+ }
+
+ for ftype, data := range contents {
+ if err = os.WriteFile(contentsPath(ftype), []byte(data), 0666); err != nil {
+ test.Error("TestNewContentFromFile setup failed:", err)
+ }
+ }
+
+ var p Page
+ for ftype := range contents {
+ if err = p.NewContentFromFile(contentsPath(ftype)); err != nil {
+ test.Fatal("NewContentFromFile failed for", ftype, err)
+ }
+ }
+}
diff --git a/page.go b/page.go
@@ -1,48 +1,16 @@
package main
import (
- "bufio"
"bytes"
- "fmt"
- "github.com/yuin/goldmark"
- goldmarkext "github.com/yuin/goldmark/extension"
- goldmarkparse "github.com/yuin/goldmark/parser"
- goldmarkhtml "github.com/yuin/goldmark/renderer/html"
- "io"
- "io/fs"
- "notabug.org/gearsix/suti"
"os"
"path/filepath"
- "sort"
+ "notabug.org/gearsix/suti"
"strings"
"time"
)
const timefmt = "2006-01-02"
-func lastFileMod(fpath string) time.Time {
- t := time.Now() // default/error ret
- if fd, e := os.Stat(fpath); e != nil {
- return t
- } else if !fd.IsDir() {
- return fd.ModTime()
- } else {
- t = fd.ModTime()
- }
-
- dir, err := os.ReadDir(fpath)
- if err != nil {
- return t
- }
-
- for i, d := range dir {
- if fd, err := d.Info(); err == nil && (i == 0 || fd.ModTime().After(t)) {
- t = fd.ModTime()
- }
- }
- return t
-}
-
func titleFromPath(path string) (title string) {
if title = filepath.Base(path); title == "/" {
title = "Home"
@@ -53,111 +21,6 @@ func titleFromPath(path string) (title string) {
return
}
-var contentExts = [6]string{
- "", // pre-formatted text
- ".txt", // plain-text
- ".html", // HTML
- ".md", // commonmark + extensions (linkify, auto-heading id, unsafe HTML)
- ".gfm", // github-flavoured markdown
- ".cm", // commonmark
-}
-
-func isContentExt(ext string) int {
- for i, supported := range contentExts {
- if ext == supported {
- return i
- }
- }
- return -1
-}
-
-// LoadPagesDir parses all files/directories in `dir` into a `Content`.
-// For each directory, a new `Page` element will be generated, any file with a
-// filetype found in `contentExts`, will be parsed into a string of HTML
-// and appended to the `.Content` of the `Page` generated for it's parent
-// directory.
-func LoadPagesDir(dir string) (p []Page, e error) {
- if _, e = os.Stat(dir); e != nil {
- return
- }
- dir = filepath.Clean(dir)
-
- pages := make(map[string]Page)
- dmetas := make(map[string]Meta)
-
- e = filepath.Walk(dir, func(fpath string, info fs.FileInfo, err error) error {
- if err != nil {
- return err
- }
- if ignoreFile(fpath) {
- return nil
- }
-
- if info.IsDir() {
- path := pagePath(dir, fpath)
- pages[path] = NewPage(path, lastFileMod(fpath))
- } else {
- path := pagePath(dir, filepath.Dir(fpath))
- page := pages[path]
-
- if suti.IsSupportedDataLang(filepath.Ext(fpath)) > -1 {
- var m Meta
- if err = suti.LoadDataFilepath(fpath, &m); err == nil {
- if strings.Contains(filepath.Base(fpath), "defaults.") ||
- strings.Contains(filepath.Base(fpath), "default.") {
- if meta, ok := dmetas[path]; ok {
- m.MergeMeta(meta, false)
- }
- dmetas[path] = m
- } else {
- page.Meta.MergeMeta(m, true)
- }
- }
- } else if isContentExt(filepath.Ext(fpath)) > -1 {
- err = page.NewContentFromFile(fpath)
- } else if suti.IsSupportedDataLang(filepath.Ext(fpath)) == -1 {
- page.Assets = append(page.Assets, filepath.Join(path, filepath.Base(fpath)))
- }
-
- pages[path] = page
- }
- return err
- })
-
- for _, page := range pages {
- page.applyDefaults(dmetas)
- p = append(p, page)
- }
-
- sort.SliceStable(p, func(i, j int) bool {
- if it, err := time.Parse(timefmt, p[i].Updated); err == nil {
- if jt, err := time.Parse(timefmt, p[j].Updated); err == nil {
- return it.After(jt)
- }
- }
- return false
- })
-
- p = BuildSitemap(p)
-
- return
-}
-
-// Meta is the structure any metadata is parsed into (_.toml_, _.json_, etc)
-type Meta map[string]interface{}
-
-// MergeMeta merges `meta` into `m`. When there are matching keys in both,
-// `overwrite` determines whether the existing value in `m` is overwritten.
-func (m Meta) MergeMeta(meta Meta, overwrite bool) {
- for k, v := range meta {
- if _, ok := m[k]; ok && overwrite {
- m[k] = v
- } else if !ok {
- m[k] = v
- }
- }
-}
-
func pagePath(root, path string) string {
path = strings.TrimPrefix(path, root)
if len(path) == 0 {
@@ -176,7 +39,7 @@ type Page struct {
Path string
Nav Nav
Meta Meta
- Contents []string
+ Contents []Content
Assets []string
Updated string
}
@@ -192,6 +55,21 @@ type Nav struct {
Crumbs []*Page
}
+// Meta is the structure any metadata is parsed into (_.toml_, _.json_, etc)
+type Meta map[string]interface{}
+
+// MergeMeta merges `meta` into `m`. When there are matching keys in both,
+// `overwrite` determines whether the existing value in `m` is overwritten.
+func (m Meta) MergeMeta(meta Meta, overwrite bool) {
+ for k, v := range meta {
+ if _, ok := m[k]; ok && overwrite {
+ m[k] = v
+ } else if !ok {
+ m[k] = v
+ }
+ }
+}
+
// NewPage returns a Page with init values. `.Path` will be set to `path`.
// Updated is set to time.Now(). Any other values will simply be initialised.
func NewPage(path string, updated time.Time) Page {
@@ -200,7 +78,7 @@ func NewPage(path string, updated time.Time) Page {
Path: path,
Nav: Nav{},
Meta: Meta{"Title": titleFromPath(path)},
- Contents: make([]string, 0),
+ Contents: make([]Content, 0),
Assets: make([]string, 0),
Updated: updated.Format(timefmt),
}
@@ -220,54 +98,6 @@ func (p *Page) TemplateName() string {
}
}
-// NewContentFromFile loads the file from `fpath` and converts it to HTML
-// from the language matching it's file extension (see below).
-// - ".txt" = plain-text
-// - ".md", ".gfm", ".cm" = various flavours of markdown
-// - ".html" = parsed as-is
-// Successful conversions are appended to `p.Contents`
-func (p *Page) NewContentFromFile(fpath string) (err error) {
- var buf []byte
- if f, err := os.Open(fpath); err == nil {
- buf, err = io.ReadAll(f)
- f.Close()
- }
- if err != nil {
- return
- }
-
- var body string
- for _, lang := range contentExts {
- if filepath.Ext(fpath) == lang {
- switch lang {
- case "":
- body = "<pre>" + string(buf) + "</pre>"
- case ".txt":
- body = convertTextToHTML(bytes.NewReader(buf))
- case ".md":
- fallthrough
- case ".gfm":
- fallthrough
- case ".cm":
- body, err = convertMarkdownToHTML(lang, buf)
- case ".html":
- body = string(buf)
- default:
- break
- }
- }
- }
- if len(body) == 0 {
- err = fmt.Errorf("invalid filetype (%s) passed to NewContentFromFile",
- filepath.Ext(fpath))
- }
- if err == nil {
- p.Contents = append(p.Contents, body)
- }
-
- return err
-}
-
func (page *Page) applyDefaults(defaultMetas map[string]Meta) {
for i, p := range page.Path {
if p != '/' {
@@ -301,96 +131,10 @@ func (p *Page) Build(outDir string, t suti.Template) (out string, err error) {
return out, err
}
-// convertTextToHTML parses textual data from `in` and line-by-line converts
-// it to HTML. Conversion rules are as follows:
-// - Blank lines (with escape characters trimmed) will close any opon tags
-// - If a text line is prefixed with a tab and no tag is open, it will open a <pre> tag
-// - Otherwise any line of text will open a <p> tag
-func convertTextToHTML(in io.Reader) (html string) {
- var tag int
- const p = 1
- const pre = 2
-
- fscan := bufio.NewScanner(in)
- for fscan.Scan() {
- line := fscan.Text()
- if len(strings.TrimSpace(line)) == 0 {
- switch tag {
- case p:
- html += "</p>\n"
- case pre:
- html += "</pre>\n"
- }
- tag = 0
- } else if tag == 0 && line[0] == '\t' {
- tag = pre
- html += "<pre>" + line[1:] + "\n"
- } else if tag == 0 || (tag == pre && line[0] != '\t') {
- if tag == pre {
- html += "</pre>\n"
- }
- tag = p
- html += "<p>" + line
- } else if tag == p {
- html += " " + line
- } else if tag == pre {
- html += line[1:] + "\n"
- }
- }
- if tag == p {
- html += "</p>"
- } else if tag == pre {
- html += "</pre>"
- }
-
- return html
-}
-
-// convertMarkdownToHTML initialises a `goldmark.Markdown` based on `lang` and
-// returns values from calling it's `Convert` function on `in`.
-// Markdown `lang` options, see the code for specfics:
-// - ".gfm" = github-flavoured markdown
-// - ".cm" = standard commonmark
-// - ".md" (and anything else) = commonmark + extensions (linkify, auto-heading id, unsafe HTML)
-func convertMarkdownToHTML(lang string, buf []byte) (string, error) {
- var markdown goldmark.Markdown
- switch lang {
- case ".gfm":
- markdown = goldmark.New(
- goldmark.WithExtensions(
- goldmarkext.GFM,
- goldmarkext.Table,
- goldmarkext.Strikethrough,
- goldmarkext.Linkify,
- goldmarkext.TaskList,
- ),
- goldmark.WithParserOptions(
- goldmarkparse.WithAutoHeadingID(),
- ),
- goldmark.WithRendererOptions(
- goldmarkhtml.WithUnsafe(),
- goldmarkhtml.WithHardWraps(),
- ),
- )
- case ".cm":
- markdown = goldmark.New()
- case ".md":
- fallthrough
- default:
- markdown = goldmark.New(
- goldmark.WithExtensions(
- goldmarkext.Linkify,
- ),
- goldmark.WithParserOptions(
- goldmarkparse.WithAutoHeadingID(),
- ),
- goldmark.WithRendererOptions(
- goldmarkhtml.WithUnsafe(),
- ),
- )
+func (p *Page) NewContentFromFile(fpath string) (err error) {
+ var c Content
+ if c, err = NewContentFromFile(fpath); err == nil {
+ p.Contents = append(p.Contents, c)
}
-
- var out bytes.Buffer
- err := markdown.Convert(buf, &out)
- return out.String(), err
+ return
}
diff --git a/page_test.go b/page_test.go
@@ -8,21 +8,6 @@ import (
"time"
)
-func TestLoadPagesDir(test *testing.T) {
- var err error
- tdir := test.TempDir()
- if err = createTestContents(tdir); err != nil {
- test.Errorf("failed to create test content: %s", err)
- }
-
- var p []Page
- if p, err = LoadPagesDir(tdir); err != nil {
- test.Fatalf("LoadPagesDir failed: %s", err)
- }
-
- validateTestPages(test, p, err)
-}
-
func TestMergeMeta(test *testing.T) {
test.Parallel()
@@ -90,37 +75,6 @@ func TestTemplateName(test *testing.T) {
}
}
-func TestNewContentFromFile(test *testing.T) {
- test.Parallel()
-
- var err error
- contents := map[string]string{
- "txt": `test`,
- "md": "**test**\ntest",
- "gfm": "**test**\ntest",
- "cm": "**test**",
- "html": `<b>test</b>`,
- }
-
- tdir := test.TempDir()
- contentsPath := func(ftype string) string {
- return tdir + "/test." + ftype
- }
-
- for ftype, data := range contents {
- if err = os.WriteFile(contentsPath(ftype), []byte(data), 0666); err != nil {
- test.Error("TestNewContentFromFile setup failed:", err)
- }
- }
-
- var p Page
- for ftype := range contents {
- if err = p.NewContentFromFile(contentsPath(ftype)); err != nil {
- test.Fatal("NewContentFromFile failed for", ftype, err)
- }
- }
-}
-
func TestCopyAssets(test *testing.T) {
test.Parallel()
diff --git a/pagr.go b/pagr.go
@@ -62,7 +62,7 @@ func main() {
vlog("loaded config: %s\n", config)
var pages []Page
- pages, err = LoadPagesDir(config.Pages)
+ pages, err = LoadContentsDir(config.Pages)
check(err)
ilog.Printf("loaded %d content pages", len(pages))
diff --git a/sitemap_test.go b/sitemap_test.go
@@ -16,7 +16,7 @@ func TestBuildSitemap(test *testing.T) {
}
var p []Page
- if p, err = LoadPagesDir(tdir); err != nil {
+ if p, err = LoadContentsDir(tdir); err != nil {
test.Errorf("LoadPagesDir failed: %s", err)
}
