fill in some content for the serve post

docs/src/blog/a-site-server-with-live-reload.org

@@ -20,7 +20,8 @@ The beauty of the ~serve~ command was that I could start with the most naive imp
 
 *** A basic file server
 
-- basic fs server implementaion
+The minimum viable implementation of the ~serve~ command consisted in rendering the site by calling ~site.Build(config)~ and serving the target site directory with a local server. Go's standard ~net/http~ already provides facilities for local file servers:
+
 #+begin_src go
 func Serve(config config.Config) error {
 	// load and build the project
@@ -37,8 +38,8 @@ func Serve(config config.Config) error {
 }
 #+end_src
 
-- improve for directory and html handling
+This only required a minor changed (which I based on [[https://stackoverflow.com/a/57281956/993769][this]] StackOverflow answer) to allow request urls to omit the ~.html~ suffix so the local server behaved as I expected a production web server would:
-  https://stackoverflow.com/a/57281956/993769
+
 #+begin_src go
 type HTMLFileSystem struct {
 	dirFS http.Dir
@@ -57,13 +58,23 @@ func (htmlFS HTMLFileSystem) Open(name string) (http.File, error) {
 }
 #+end_src
 
-#+begin_src go
+The ~HTMLFileSystem~ above wraps the standard ~http.Dir~ optionally looking for e.g. ~target/blog/hello.html~ when the URL requests for ~/blog/hello~. The server setup thus changed to:
-fs := http.FileServer(HTMLFileSystem{http.Dir(config.TargetDir)})
+
-http.Handle("/", fs)
+#+begin_src diff
+-	fs := http.FileServer(HTMLFileSystem{http.Dir(config.TargetDir)})
++	fs := http.FileServer(http.Dir(config.TargetDir))
+	http.Handle("/", fs)
+
+	fmt.Println("server listening at http://localhost:4001/")
+	return http.ListenAndServe(":4001", nil)
 #+end_src
 
 *** Watching for changes
-- fsnotify to trigger builds
+The obvious next step was to, instead of building the site once before starting the server, watching the project source directory and trigger new site builds every time a file change was detected.
+
+I found the [[https://github.com/fsnotify/fsnotify][fsnotify]] library for this exact purpose; the fact that both Hugo and gojekyll listed it in their dependencies hinted to me that it was a reasonable choice for job.
+
+Following the [[https://github.com/fsnotify/fsnotify#usage][example]] in the documentation, I created a watcher and a goroutine that reacted with a ~site.Build~ call to every incoming event:
 
 #+begin_src go
 func runWatcher(config *config.Config) {
@@ -81,13 +92,15 @@ func runWatcher(config *config.Config) {
 		}
 	}()
 }
+#+end_src
 
-// Configure the given watcher to notify for changes
+Then made the watcher look at changes in the project ~src~ directory:
-// in the project source files
+
+#+begin_src go
 func watchProjectFiles(watcher *fsnotify.Watcher, config *config.Config) {
-	// fsnotify watches all files within a dir, but non recursively
+	// fsnotify watches all files within a dir, but non-recursively
 	// this walks through the src dir and adds watches for each found directory
-	filepath.WalkDir(SRC_DIR, func(path string, entry fs.DirEntry, err error) error {
+	filepath.WalkDir(config.SrcDir, func(path string, entry fs.DirEntry, err error) error {
 		if entry.IsDir() {
 			watcher.Add(path)
 		}
@@ -96,41 +109,17 @@ func watchProjectFiles(watcher *fsnotify.Watcher, config *config.Config) {
 }
 #+end_src
 
-- delay to prevent bursts
-
-#+begin_src diff
-func runWatcher(config *config.Config) {
-	watcher, _ := fsnotify.NewWatcher()
--	defer watchProjectFiles(watcher, config)
-+
-+	rebuildAfter := time.AfterFunc(0, func() {
-+		watchProjectFiles(watcher, config)
-+		site.Build(*config)
-+	})
-
-	go func() {
-		for event := range watcher.Events {
-			fmt.Printf("file %s changed\n", event.Name)
-
--			watchProjectFiles(watcher, config)
--			site.Build(*config)
-+			// Schedule a rebuild to trigger after a delay.
-+			// If there was another one pending it will be canceled.
-+			rebuildAfter.Stop()
-+			rebuildAfter.Reset(100 * time.Millisecond)
-
-
-		}
-	}()
-}
-#+end_src
-
 *** Build optimizations
-- optimization: ln static files
+At this point the file server was useful, always responding with the most recent version of the site. But the responsiveness of the command was less than ideal: the entire website had to be processed and copied to the target for every file save in the source.
-- optimization: worker pool
+
+I wanted to make some performance improvements to this process, but without adding much code complexity: instead of getting into incremental or conditional builds, I wanted to keep building the entire site on very change, only faster.
+
+The first cheap optimization was obvious from looking at the command output: most of the work was copying static assets (e.g. images, static CSS files, etc.). So I changed the ~site.Build~ implementation to optionally create links instead of copying files.
+
+The next thing I wanted to try was to process source files work concurrently. The logic of the target building was handled by a method from an internal ~site~ struct:
 
 #+begin_src go
-func (site *Site) Build() error {
+func (site *site) build() error {
 	// clear previous target contents
 	os.RemoveAll(site.Config.TargetDir)
 
@@ -150,6 +139,8 @@ func (site *Site) Build() error {
 }
 #+end_src
 
+The ~build~ method walks the source file tree, recreating directories in the target. For non-directory files, it delegates the actual file processing (rendering templates, converting markdown and org-mode syntax to HTML, "smartifying" quotes, and copying the results to the target files) to another internal method: ~site.buildFile~. I wanted this one to run in a worker pool; I found the facilities I needed in a couple of [[https://gobyexample.com/][Go by Example]] entries:
+
 #+begin_src go
 // Create a channel to send paths to build and a worker pool to handle them concurrently
 func spawnBuildWorkers(site *site) (*sync.WaitGroup, chan string) {
@@ -169,6 +160,10 @@ func spawnBuildWorkers(site *site) (*sync.WaitGroup, chan string) {
 }
 #+end_src
 
+The function above creates a buffered channel to receive source file paths, and a worker pool of the size of the available CPU cores. Each worker registers itself on a ~WaitGroup~ that can be used by callers to block until all workers finish their work.
+
+Then, it was just a matter of creating the workers and sending the filepaths through the channel instead of building the files sequentially:
+
 #+begin_src diff
 func (site *site) build() error {
 	// clear previous target contents
@@ -198,7 +193,9 @@ func (site *site) build() error {
 }
 #+end_src
 
-- in other languages, a similar change would have required adding async/await statements on half of my codebase.
+The ~defer close(files)~ closes the channel to inform the workers that no more work will be sent, and the ~defer wg.Wait()~ blocks until all finish processing what they read from the channel.
+
+I loved that I could turn a sequential piece of code into a concurrent one with minimal structural changes, without touching calling sites of the affected function. In other languages, a similar process would have required me to add ~async~ and ~await~ statements to half of the codebase.
 
 *** Live reload
 
@@ -326,27 +323,52 @@ func (broker *EventBroker) publish(event string)
 -func runWatcher(config *config.Config) {
 +func runWatcher(config *config.Config) *EventBroker {
 	watcher, _ := fsnotify.NewWatcher()
+	defer watchProjectFiles(watcher, config)
 +	broker := newEventBroker()
 
-	rebuildAfter := time.AfterFunc(0, func() {
+	go func() {
+		for event := range watcher.Events {
+		fmt.Printf("file %s changed\n", event.Name)
+
+		// new src directories could be triggering this event
+		// so project files need to be re-added every time
 		watchProjectFiles(watcher, config)
 		site.Build(*config)
 +		broker.publish("rebuild")
-	})
+		}
+	}()
++	return broker
+}
+#+end_src
 
 
+** Preventing bursts
+
+#+begin_src diff
+func runWatcher(config *config.Config) *EventBroker {
+	watcher, _ := fsnotify.NewWatcher()
+-	defer watchProjectFiles(watcher, config)
+	broker := newEventBroker()
+
++	rebuildAfter := time.AfterFunc(0, func() {
++		watchProjectFiles(watcher, config)
++		site.Build(*config)
++		broker.publish("rebuild")
++	})
+
 	go func() {
 		for event := range watcher.Events {
 			fmt.Printf("file %s changed\n", event.Name)
 
-			// Schedule a rebuild to trigger after a delay.
+-			watchProjectFiles(watcher, config)
-			// If there was another one pending it will be canceled.
+-			site.Build(*config)
-			rebuildAfter.Stop()
+-			broker.publish("rebuild")
-			rebuildAfter.Reset(100 * time.Millisecond)
++			// Schedule a rebuild to trigger after a delay.
-
++			// If there was another one pending it will be canceled.
++			rebuildAfter.Stop()
++			rebuildAfter.Reset(100 * time.Millisecond)
 		}
 	}()
-
+	return broker
-+	return broker
 }
 #+end_src