Miroirs/jorge
1
0
Fork 0
mirror of https://github.com/facundoolano/jorge.git synced 2024-11-16 07:47:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

build command refactors (#1)

Browse source 
* extract buildTarget function

* first stab at build target rendering refactor

* more cleanup

* properly separate parse and render stages of liquid templates

* move some more rendering from commands to site

* move org rendering again, to fix bug

* revert more unsatisfactory changes

* move build to site

* markdown support

* remove comments

* fix tests

* markdown test

* cleanup scanner bytes usage

* reuse the template engine
This commit is contained in:
Facundo Olano 2024-02-14 23:54:46 -03:00 committed by GitHub
parent b181e3d855
commit e5917dc21e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 218 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
commands/commands.go
View file

@ -2,11 +2,6 @@ package commands
import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/facundoolano/blorg/site"
)
@ -14,7 +9,6 @@ import (
const SRC_DIR = "src"
const TARGET_DIR = "target"
const LAYOUTS_DIR = "layouts"
const FILE_RW_MODE = 0777
func Init() error {
// get working directory
@ -25,6 +19,16 @@ func Init() error {
return nil
}
func New() error {
// prompt for title
// slugify
// fail if file already exist
// create a new .org file with the slug
// add front matter and org options
fmt.Println("not implemented yet")
return nil
}
// Read the files in src/ render them and copy the result to target/
// TODO add root dir override support
func Build() error {
@ -33,69 +37,5 @@ func Build() error {
return err
}
// clear previous target contents
os.RemoveAll(TARGET_DIR)
os.Mkdir(TARGET_DIR, FILE_RW_MODE)
// walk the source directory, creating directories and files at the target dir
return filepath.WalkDir(SRC_DIR, func(path string, entry fs.DirEntry, err error) error {
subpath, _ := filepath.Rel(SRC_DIR, path)
targetPath := filepath.Join(TARGET_DIR, subpath)
if entry.IsDir() {
os.MkdirAll(targetPath, FILE_RW_MODE)
} else {
if templ, ok := site.Templates[path]; ok {
// if a template was found at source, render it
content, err := site.Render(templ)
if err != nil {
return err
}
// write the file contents over to target at the same location
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath)) + templ.Ext()
fmt.Println("writing", targetPath)
return os.WriteFile(targetPath, []byte(content), FILE_RW_MODE)
} else {
// if a non template was found, copy file as is
fmt.Println("writing", targetPath)
return copyFile(path, targetPath)
}
}
return nil
})
}
func copyFile(source string, target string) error {
// does this need to be so verbose?
srcFile, err := os.Open(source)
if err != nil {
return err
}
defer srcFile.Close()
targetFile, _ := os.Create(target)
if err != nil {
return err
}
defer targetFile.Close()
_, err = io.Copy(targetFile, srcFile)
if err != nil {
return err
}
return targetFile.Sync()
}
func New() error {
// prompt for title
// slugify
// fail if file already exist
// create a new .org file with the slug
// add front matter and org options
fmt.Println("not implemented yet")
return nil
return site.Build(SRC_DIR, TARGET_DIR, true, false)
}

15
commands/serve.go
View file

@ -8,20 +8,23 @@ import (
"path/filepath"
"strings"
"github.com/facundoolano/blorg/site"
"github.com/fsnotify/fsnotify"
)
// Generate and serve the site, rebuilding when the source files change.
func Serve() error {
// TODO tweak the building logic to inject js snippet that reloads the browser on rebuild
site, err := site.Load(SRC_DIR, LAYOUTS_DIR)
if err != nil {
return err
}
// first rebuild the target
if err := Build(); err != nil {
if err := site.Build(SRC_DIR, TARGET_DIR, false, true); err != nil {
return err
}
// watch for changes in src and layouts, and trigger a rebuild
watcher, err := setupWatcher()
watcher, err := setupWatcher(site)
if err != nil {
return err
}
@ -54,7 +57,7 @@ func (d HTMLDir) Open(name string) (http.File, error) {
return f, err
}
func setupWatcher() (*fsnotify.Watcher, error) {
func setupWatcher(site *site.Site) (*fsnotify.Watcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
@ -85,7 +88,7 @@ func setupWatcher() (*fsnotify.Watcher, error) {
return
}
if err := Build(); err != nil {
if err := site.Build(SRC_DIR, TARGET_DIR, false, true); err != nil {
fmt.Println("error:", err)
return
}

6
go.mod
View file

@ -3,15 +3,15 @@ module github.com/facundoolano/blorg
go 1.22.0
require (
github.com/fsnotify/fsnotify v1.7.0
github.com/niklasfasching/go-org v1.7.0
gopkg.in/osteele/liquid.v1 v1.2.4
github.com/osteele/liquid v1.3.2
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/osteele/liquid v1.3.2 // indirect
github.com/osteele/tuesday v1.0.3 // indirect
github.com/yuin/goldmark v1.7.0 // indirect
golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
golang.org/x/sys v0.4.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect

4
go.sum
View file

@ -12,6 +12,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA=
github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -22,8 +24,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/osteele/liquid.v1 v1.2.4 h1:OioNeCaVyWL1jRXzRqQ2vr4ISBbTgtnYsJeVlToLhBw=
gopkg.in/osteele/liquid.v1 v1.2.4/go.mod h1:9Bx5f04tf9SVwv3Tcx93dx3WH0EKWmE0Gjd6Dyoc5cs=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

103
site/site.go
View file

@ -1,7 +1,9 @@
package site
import (
"bytes"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
@ -12,8 +14,8 @@ import (
"github.com/facundoolano/blorg/templates"
)
// TODO review build and other commands and think what can be brought over here.
// e.g. SRC and TARGET dir knowledge
const FILE_RW_MODE = 0777
type Site struct {
config map[string]string // may need to make this interface{} if config gets sophisticated
layouts map[string]templates.Template
@ -21,16 +23,18 @@ type Site struct {
pages []map[string]interface{}
tags map[string][]map[string]interface{}
Templates map[string]*templates.Template
templateEngine *templates.Engine
templates map[string]*templates.Template
}
func Load(srcDir string, layoutsDir string) (*Site, error) {
// TODO load config from config.yml
site := Site{
layouts: make(map[string]templates.Template),
Templates: make(map[string]*templates.Template),
config: make(map[string]string),
tags: make(map[string][]map[string]interface{}),
layouts: make(map[string]templates.Template),
templates: make(map[string]*templates.Template),
config: make(map[string]string),
tags: make(map[string][]map[string]interface{}),
templateEngine: templates.NewEngine(),
}
if err := site.loadLayouts(layoutsDir); err != nil {
@ -57,7 +61,7 @@ func (site *Site) loadLayouts(layoutsDir string) error {
if !entry.IsDir() {
filename := entry.Name()
path := filepath.Join(layoutsDir, filename)
templ, err := templates.Parse(path)
templ, err := templates.Parse(site.templateEngine, path)
if err != nil {
return err
}
@ -80,7 +84,7 @@ func (site *Site) loadTemplates(srcDir string) error {
err = filepath.WalkDir(srcDir, func(path string, entry fs.DirEntry, err error) error {
if !entry.IsDir() {
templ, err := templates.Parse(path)
templ, err := templates.Parse(site.templateEngine, path)
// if sometime fails or this is not a template, skip
if err != nil || templ == nil {
return err
@ -112,7 +116,7 @@ func (site *Site) loadTemplates(srcDir string) error {
site.pages = append(site.pages, templ.Metadata)
}
}
site.Templates[path] = templ
site.templates[path] = templ
}
return nil
})
@ -132,7 +136,63 @@ func (site *Site) loadTemplates(srcDir string) error {
return nil
}
func (site Site) Render(templ *templates.Template) (string, error) {
// TODO consider making minify and reload site.config values
func (site *Site) Build(srcDir string, targetDir string, minify bool, htmlReload bool) error {
// clear previous target contents
os.RemoveAll(targetDir)
os.Mkdir(srcDir, FILE_RW_MODE)
// walk the source directory, creating directories and files at the target dir
return filepath.WalkDir(srcDir, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
subpath, _ := filepath.Rel(srcDir, path)
targetPath := filepath.Join(targetDir, subpath)
// if it's a directory, just create the same at the target
if entry.IsDir() {
return os.MkdirAll(targetPath, FILE_RW_MODE)
}
var contentReader io.Reader
if templ, found := site.templates[path]; found {
content, err := site.render(templ)
if err != nil {
return err
}
targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath)) + templ.Ext()
contentReader = bytes.NewReader(content)
} else {
// if no template found at location, treat the file as static
// write its contents to target
srcFile, err := os.Open(path)
if err != nil {
return err
}
defer srcFile.Close()
contentReader = srcFile
}
targetExt := filepath.Ext(targetPath)
// if live reload is enabled, inject the reload snippet to html files
if htmlReload && targetExt == ".html" {
// TODO inject live reload snippet
}
// if enabled, minify web files
if minify && (targetExt == ".html" || targetExt == ".css" || targetExt == ".js") {
// TODO minify output
}
// write the file contents over to target
fmt.Println("writing", targetPath)
return writeToFile(targetPath, contentReader)
})
}
func (site Site) render(templ *templates.Template) ([]byte, error) {
ctx := map[string]interface{}{
"site": map[string]interface{}{
"config": site.config,
@ -145,7 +205,7 @@ func (site Site) Render(templ *templates.Template) (string, error) {
ctx["page"] = templ.Metadata
content, err := templ.Render(ctx)
if err != nil {
return "", err
return nil, err
}
// recursively render parent layouts
@ -157,9 +217,24 @@ func (site Site) Render(templ *templates.Template) (string, error) {
content, err = layout_templ.Render(ctx)
layout = layout_templ.Metadata["layout"]
} else {
return "", fmt.Errorf("layout '%s' not found", layout)
return nil, fmt.Errorf("layout '%s' not found", layout)
}
}
return content, err
return content, nil
}
func writeToFile(targetPath string, source io.Reader) error {
targetFile, err := os.Create(targetPath)
if err != nil {
return err
}
defer targetFile.Close()
_, err = io.Copy(targetFile, source)
if err != nil {
return err
}
return targetFile.Sync()
}

24
site/site_test.go
View file

@ -81,9 +81,9 @@ title: about
_, ok = site.layouts["post"]
assert(t, ok)
content, err = site.Render(site.Templates[helloPath])
output, err := site.render(site.templates[helloPath])
assertEqual(t, err, nil)
assertEqual(t, content, `<html>
assertEqual(t, string(output), `<html>
<head><title>hello world!</title></head>
<body>
<h1>hello world!</h1>
@ -92,9 +92,9 @@ title: about
</body>
</html>`)
content, err = site.Render(site.Templates[goodbyePath])
output, err = site.render(site.templates[goodbyePath])
assertEqual(t, err, nil)
assertEqual(t, content, `<html>
assertEqual(t, string(output), `<html>
<head><title>goodbye!</title></head>
<body>
<h1>goodbye!</h1>
@ -103,9 +103,9 @@ title: about
</body>
</html>`)
content, err = site.Render(site.Templates[aboutPath])
output, err = site.render(site.templates[aboutPath])
assertEqual(t, err, nil)
assertEqual(t, content, `<html>
assertEqual(t, string(output), `<html>
<head><title>about</title></head>
<body>
<p>about this site</p>
@ -153,9 +153,9 @@ date: 2023-01-01
defer os.Remove(file.Name())
site, err := Load(src, layouts)
content, err = site.Render(site.Templates[file.Name()])
output, err := site.render(site.templates[file.Name()])
assertEqual(t, err, nil)
assertEqual(t, content, `<ul>
assertEqual(t, string(output), `<ul>
<li>2024-02-01 <a href="/goodbye">goodbye!</a></li>
<li>2024-01-01 <a href="/hello">hello world!</a></li>
<li>2023-01-01 <a href="/an-oldie">an oldie!</a></li>
@ -206,9 +206,9 @@ tags: [software]
defer os.Remove(file.Name())
site, err := Load(src, layouts)
content, err = site.Render(site.Templates[file.Name()])
output, err := site.render(site.templates[file.Name()])
assertEqual(t, err, nil)
assertEqual(t, content, `<h1>software</h1>
assertEqual(t, string(output), `<h1>software</h1>
hello world!
an oldie!
@ -257,9 +257,9 @@ title: "2. an oldie!"
defer os.Remove(file.Name())
site, err := Load(src, layouts)
content, err = site.Render(site.Templates[file.Name()])
output, err := site.render(site.templates[file.Name()])
assertEqual(t, err, nil)
assertEqual(t, content, `<ul>
assertEqual(t, string(output), `<ul>
<li><a href="/01-hello">1. hello world!</a></li>
<li><a href="/02-an-oldie">2. an oldie!</a></li>
<li><a href="/03-goodbye">3. goodbye!</a></li>

99
templates/templates.go
View file

@ -3,6 +3,7 @@ package templates
import (
"bufio"
"bytes"
"errors"
"fmt"
"os"
@ -10,18 +11,26 @@ import (
"strings"
"github.com/niklasfasching/go-org/org"
"gopkg.in/osteele/liquid.v1"
"github.com/osteele/liquid"
"github.com/yuin/goldmark"
"gopkg.in/yaml.v3"
)
const FM_SEPARATOR = "---"
type Engine = liquid.Engine
type Template struct {
SrcPath string
Metadata map[string]interface{}
SrcPath string
Metadata map[string]interface{}
liquidTemplate liquid.Template
}
func Parse(path string) (*Template, error) {
func NewEngine() *Engine {
return liquid.NewEngine()
}
func Parse(engine *Engine, path string) (*Template, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
@ -37,18 +46,25 @@ func Parse(path string) (*Template, error) {
return nil, nil
}
// read and parse the yaml from the front matter
// extract the yaml front matter and save the rest of the template content separately
var yamlContent []byte
closed := false
var liquidContent []byte
yamlClosed := false
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == FM_SEPARATOR {
closed = true
break
line := append(scanner.Bytes(), '\n')
if yamlClosed {
liquidContent = append(liquidContent, line...)
} else {
if strings.TrimSpace(scanner.Text()) == FM_SEPARATOR {
yamlClosed = true
continue
}
yamlContent = append(yamlContent, line...)
}
yamlContent = append(yamlContent, []byte(line+"\n")...)
}
if !closed {
liquidContent = bytes.TrimSuffix(liquidContent, []byte("\n"))
if !yamlClosed {
return nil, errors.New("front matter not closed")
}
@ -60,50 +76,45 @@ func Parse(path string) (*Template, error) {
}
}
templ := Template{SrcPath: path, Metadata: metadata}
liquid, err := engine.ParseTemplateAndCache(liquidContent, path, 0)
if err != nil {
return nil, err
}
templ := Template{SrcPath: path, Metadata: metadata, liquidTemplate: *liquid}
return &templ, nil
}
// Return the extension for the output format of this template
func (templ Template) Ext() string {
ext := filepath.Ext(templ.SrcPath)
if ext == ".org" {
ext = ".html"
if ext == ".org" || ext == ".md" {
return ".html"
}
return ext
}
func (templ Template) Render(context map[string]interface{}) (string, error) {
file, _ := os.Open(templ.SrcPath)
defer file.Close()
scanner := bufio.NewScanner(file)
// first line is the front matter delimiter, Scan to skip
// and keep skipping until the closing delimiter
scanner.Scan()
scanner.Scan()
for scanner.Text() != FM_SEPARATOR {
scanner.Scan()
func (templ Template) Render(context map[string]interface{}) ([]byte, error) {
content, err := templ.liquidTemplate.Render(context)
if err != nil {
return nil, err
}
// now read the proper template contents to memory
contents := ""
isFirstLine := true
for scanner.Scan() {
if isFirstLine {
isFirstLine = false
contents = scanner.Text()
} else {
contents += "\n" + scanner.Text()
ext := filepath.Ext(templ.SrcPath)
if ext == ".org" {
doc := org.New().Parse(bytes.NewReader(content), templ.SrcPath)
contentStr, err := doc.Write(org.NewHTMLWriter())
if err != nil {
return nil, err
}
content = []byte(contentStr)
} else if ext == ".md" {
var buf bytes.Buffer
if err := goldmark.Convert(content, &buf); err != nil {
return nil, err
}
content = buf.Bytes()
}
if strings.HasSuffix(templ.SrcPath, ".org") {
// if it's an org file, convert to html
doc := org.New().Parse(strings.NewReader(contents), templ.SrcPath)
return doc.Write(org.NewHTMLWriter())
}
// for other file types, assume a liquid template
engine := liquid.NewEngine()
return engine.ParseAndRenderString(contents, context)
return content, nil
}

46
templates/templates_test.go
View file

@ -18,10 +18,9 @@ tags: ["software", "web"]
file := newFile("test*.html", input)
defer os.Remove(file.Name())
templ, err := Parse(file.Name())
templ, err := Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
assertEqual(t, templ.Ext(), ".html")
assertEqual(t, templ.Metadata["title"], "my new post")
assertEqual(t, templ.Metadata["subtitle"], "a blog post")
assertEqual(t, templ.Metadata["tags"].([]interface{})[0], "software")
@ -43,7 +42,7 @@ subtitle: a blog post
file := newFile("test*.html", input)
defer os.Remove(file.Name())
_, err := Parse(file.Name())
_, err := Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
// not first thing in file, leaving as is
@ -58,7 +57,7 @@ tags: ["software", "web"]
file = newFile("test*.html", input)
defer os.Remove(file.Name())
_, err = Parse(file.Name())
_, err = Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
}
@ -70,7 +69,7 @@ tags: ["software", "web"]
`
file := newFile("test*.html", input)
defer os.Remove(file.Name())
_, err := Parse(file.Name())
_, err := Parse(NewEngine(), file.Name())
assertEqual(t, err.Error(), "front matter not closed")
@ -82,7 +81,7 @@ tags: ["software", "web"]
file = newFile("test*.html", input)
defer os.Remove(file.Name())
_, err = Parse(file.Name())
_, err = Parse(NewEngine(), file.Name())
assert(t, strings.Contains(err.Error(), "invalid yaml"))
}
@ -101,7 +100,7 @@ tags: ["software", "web"]
file := newFile("test*.html", input)
defer os.Remove(file.Name())
templ, err := Parse(file.Name())
templ, err := Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
ctx := map[string]interface{}{"page": templ.Metadata}
content, err := templ.Render(ctx)
@ -131,9 +130,8 @@ tags: ["software", "web"]
file := newFile("test*.org", input)
defer os.Remove(file.Name())
templ, err := Parse(file.Name())
templ, err := Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
assertEqual(t, templ.Ext(), ".html")
content, err := templ.Render(nil)
assertEqual(t, err, nil)
@ -159,6 +157,36 @@ my Subtitle
assertEqual(t, string(content), expected)
}
func TestRenderMarkdown(t *testing.T) {
input := `---
title: my new post
subtitle: a blog post
tags: ["software", "web"]
---
# My title
## my Subtitle
- list 1
- list 2
`
file := newFile("test*.md", input)
defer os.Remove(file.Name())
templ, err := Parse(NewEngine(), file.Name())
assertEqual(t, err, nil)
content, err := templ.Render(nil)
assertEqual(t, err, nil)
expected := `<h1>My title</h1>
<h2>my Subtitle</h2>
<ul>
<li>list 1</li>
<li>list 2</li>
</ul>
`
assertEqual(t, string(content), expected)
}
// ------ HELPERS --------
func newFile(path string, contents string) *os.File {