From ccc8bc5e99d7724cad4e9e3d62217ed9555137f8 Mon Sep 17 00:00:00 2001 From: Facundo Olano Date: Sat, 17 Feb 2024 21:13:50 -0300 Subject: [PATCH] Improve local server (#6) * don't interrupt watcher on error * don't halt build on missing file * extract buildFile function * build the site with concurrent workers * improve fsnotify event treatment and output --- commands/serve.go | 23 ++++---- site/site.go | 133 +++++++++++++++++++++++++++++++--------------- 2 files changed, 100 insertions(+), 56 deletions(-) diff --git a/commands/serve.go b/commands/serve.go index cfe491b..7cdf0a8 100644 --- a/commands/serve.go +++ b/commands/serve.go @@ -1,7 +1,6 @@ package commands import ( - "errors" "fmt" "io/fs" "net/http" @@ -86,15 +85,9 @@ func setupWatcher(config *config.Config) (*fsnotify.Watcher, error) { return } - // some events can be received for temporary files, e.g. .#backup files generated by emacs while editing .org - // when events regarding such files arrive, discard them here instead of triggering a faulty rebuild - if _, err := os.Stat(event.Name); errors.Is(err, os.ErrNotExist) { - // FIXME change to log debug - fmt.Println("ignoring temporary file", event.Name) - continue - } - // chmod events are noisy, ignore them - if !event.Has(fsnotify.Chmod) { + // chmod events are noisy, ignore them. also skip create events + // which we assume meaningless until the write that comes next + if event.Has(fsnotify.Chmod) || event.Has(fsnotify.Create) { continue } @@ -103,15 +96,17 @@ func setupWatcher(config *config.Config) (*fsnotify.Watcher, error) { // since new nested directories could be triggering this change, and we need to watch those too // and since re-watching files is a noop, I just re-add the entire src everytime there's a change if err := addAll(watcher, config); err != nil { - fmt.Println("error:", err) - return + fmt.Println("couldn't add watchers:", err) + continue } if err := rebuild(config); err != nil { - fmt.Println("error:", err) - return + fmt.Println("build error:", err) + continue } + fmt.Println("done\nserver listening at", config.SiteUrl) + case err, ok := <-watcher.Errors: if !ok { return diff --git a/site/site.go b/site/site.go index ea16367..3b13ae5 100644 --- a/site/site.go +++ b/site/site.go @@ -7,8 +7,10 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "slices" "strings" + "sync" "time" "github.com/facundoolano/jorge/config" @@ -131,7 +133,7 @@ func (site *Site) loadTemplates() error { templ, err := templates.Parse(site.templateEngine, path) // if something fails or this is not a template, skip if err != nil || templ == nil { - return err + return checkFileError(err) } // set site related (?) metadata. Not sure if this should go elsewhere @@ -191,8 +193,12 @@ func (site *Site) Build() error { os.RemoveAll(site.Config.TargetDir) os.Mkdir(site.Config.SrcDir, FILE_RW_MODE) + wg, files := spawnBuildWorkers(site) + defer wg.Wait() + defer close(files) + // walk the source directory, creating directories and files at the target dir - return filepath.WalkDir(site.Config.SrcDir, func(path string, entry fs.DirEntry, err error) error { + err := filepath.WalkDir(site.Config.SrcDir, func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } @@ -203,49 +209,78 @@ func (site *Site) Build() error { if entry.IsDir() { return os.MkdirAll(targetPath, FILE_RW_MODE) } - - var contentReader io.Reader - templ, found := site.templates[path] - if !found { - // if no template found at location, treat the file as static write its contents to target - if site.Config.LinkStatic { - // dev optimization: link static files instead of copying them - abs, _ := filepath.Abs(path) - return os.Symlink(abs, targetPath) - } - - srcFile, err := os.Open(path) - if err != nil { - return err - } - defer srcFile.Close() - contentReader = srcFile - } else { - content, err := site.render(templ) - if err != nil { - return err - } - - targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath)) + templ.Ext() - contentReader = bytes.NewReader(content) - } - - targetExt := filepath.Ext(targetPath) - // if live reload is enabled, inject the reload snippet to html files - if site.Config.LiveReload && targetExt == ".html" { - // TODO inject live reload snippet - } - - // if enabled, minify web files - contentReader = site.minify(targetExt, contentReader) - - // write the file contents over to target - fmt.Println("writing", targetPath) - return writeToFile(targetPath, contentReader) + // if it's a file (either static or template) send the path to a worker to build in target + files <- path + return nil }) + + return err } -func (site Site) render(templ *templates.Template) ([]byte, error) { +// Create a channel to send paths to build and a worker pool to handle them concurrently +func spawnBuildWorkers(site *Site) (*sync.WaitGroup, chan string) { + + var wg sync.WaitGroup + files := make(chan string, 20) + + for range runtime.NumCPU() { + wg.Add(1) + go func(files <-chan string) { + defer wg.Done() + for path := range files { + site.buildFile(path) + } + }(files) + } + return &wg, files +} + +func (site *Site) buildFile(path string) error { + subpath, _ := filepath.Rel(site.Config.SrcDir, path) + targetPath := filepath.Join(site.Config.TargetDir, subpath) + + var contentReader io.Reader + var err error + templ, found := site.templates[path] + if !found { + // if no template found at location, treat the file as static write its contents to target + if site.Config.LinkStatic { + // dev optimization: link static files instead of copying them + abs, _ := filepath.Abs(path) + err = os.Symlink(abs, targetPath) + return checkFileError(err) + } + + srcFile, err := os.Open(path) + if err != nil { + return checkFileError(err) + } + defer srcFile.Close() + contentReader = srcFile + } else { + content, err := site.render(templ) + if err != nil { + return err + } + + targetPath = strings.TrimSuffix(targetPath, filepath.Ext(targetPath)) + templ.Ext() + contentReader = bytes.NewReader(content) + } + + targetExt := filepath.Ext(targetPath) + // if live reload is enabled, inject the reload snippet to html files + if site.Config.LiveReload && targetExt == ".html" { + // TODO inject live reload snippet + } + + // if enabled, minify web files + contentReader = site.minify(targetExt, contentReader) + + // write the file contents over to target + return writeToFile(targetPath, contentReader) +} + +func (site *Site) render(templ *templates.Template) ([]byte, error) { ctx := map[string]interface{}{ "site": map[string]interface{}{ "config": site.Config.AsContext(), @@ -281,6 +316,19 @@ func (site Site) render(templ *templates.Template) ([]byte, error) { return content, nil } +func checkFileError(err error) error { + // When walking the source dir it can happen that a file is present when walking starts + // but missing or inaccessible when trying to open it (this is particularly frequent with + // backup files from emacs and when running the dev server). We don't want to halt the build + // process in that situation, just inform and continue. + if os.IsNotExist(err) { + // don't abort on missing files, usually spurious temps + fmt.Println("skipping missing file", err) + return nil + } + return err +} + func writeToFile(targetPath string, source io.Reader) error { targetFile, err := os.Create(targetPath) if err != nil { @@ -293,6 +341,7 @@ func writeToFile(targetPath string, source io.Reader) error { return err } + fmt.Println("added", targetPath) return targetFile.Sync() }