commit 59a830aadfaea7dd977db018d88ccf7e006b82a2 Author: Trevor Slocum Date: Wed Feb 5 09:28:02 2020 -0800 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95b815f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea/ +dist/ +vendor/ +*.sh +godoc-static diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..1aab122 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: golang:latest + +stages: + - validate + - build + +fmt: + stage: validate + script: + - gofmt -l -s -e . + - exit $(gofmt -l -s -e . | wc -l) + +vet: + stage: validate + script: + - go vet -composites=false ./... + +test: + stage: validate + script: + - go test -race -v ./... + +build: + stage: build + script: + - go build diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..8da6a6f --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,2 @@ +0.1.0: +- Initial release diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7c20f89 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Trevor Slocum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0bc4091 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# godoc-static +[![GoDoc](https://gitlab.com/tslocum/godoc-static/raw/master/badge.svg)](https://docs.rocketnine.space/gitlab.com/tslocum/godoc-static) +[![CI status](https://gitlab.com/tslocum/godoc-static/badges/master/pipeline.svg)](https://gitlab.com/tslocum/godoc-static/commits/master) +[![Donate](https://img.shields.io/liberapay/receives/rocketnine.space.svg?logo=liberapay)](https://liberapay.com/rocketnine.space) + +Generate static Go documentation + +## Demo + +[Rocket Nine Labs Documentation](https://docs.rocketnine.space) + +## Installation + +Install `godoc-static`: + +`go get gitlab.com/tslocum/godoc-static` + +Also install `godoc`: + +`go get golang.org/x/tools/cmd/godoc` + +## Documentation + +Execute `godoc-static` with the `-help` flag for more information. + +### Usage examples + +Generate documentation for `archive`, `fmt` and `net/http` targeting `https://docs.rocketnine.space`: + +`godoc-static -base-path=/ -site-name="Rocket Nine Labs Documentation" -site-description="Welcome!" -out=/home/user/sites/docs archive fmt net/http` + +Targeting `https://rocketnine.space/docs/`: + +`godoc-static -base-path=/docs/ -site-name="Rocket Nine Labs Documentation" -site-description-file=/home/user/sitefiles/description.md -out=/home/user/sites/docs archive fmt net/http` + +## Support + +Please share issues/suggestions [here](https://gitlab.com/tslocum/godoc-static/issues). diff --git a/badge.svg b/badge.svg new file mode 100755 index 0000000..3952e41 --- /dev/null +++ b/badge.svg @@ -0,0 +1 @@ +godocgodocreferencereference \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1c55b34 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module gitlab.com/tslocum/godoc-static + +go 1.13 + +require ( + github.com/PuerkitoBio/goquery v1.5.1 + github.com/yuin/goldmark v1.1.22 + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1f40cb4 --- /dev/null +++ b/go.sum @@ -0,0 +1,12 @@ +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/yuin/goldmark v1.1.22 h1:0e0f6Zee9SAQ5yOZGNMWaOxqVvcc/9/kUWu/Kl91Jk8= +github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..1b1e8a9 --- /dev/null +++ b/main.go @@ -0,0 +1,354 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + gmhtml "github.com/yuin/goldmark/renderer/html" + "golang.org/x/net/html" +) + +const additionalCSS = ` +details { margin-top: 20px; } +summary { margin-left: 20px; cursor: pointer; } +` + +var ( + listenAddress string + basePath string + siteName string + siteDescription string + siteDescriptionFile string + linkIndex bool + outDir string + verbose bool +) + +func main() { + flag.StringVar(&listenAddress, "listen-address", "localhost:9001", "address for godoc to listen on while scraping pages") + flag.StringVar(&basePath, "base-path", "/", "site relative URL path with trailing slash") + flag.StringVar(&siteName, "site-name", "Documentation", "site name") + flag.StringVar(&siteDescription, "site-description", "", "site description (markdown-enabled)") + flag.StringVar(&siteDescriptionFile, "site-description-file", "", "path to markdown file containing site description") + flag.BoolVar(&linkIndex, "link-index", false, "set link targets to index.html instead of folder") + flag.StringVar(&outDir, "out", "", "site directory") + flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") + flag.Parse() + + var buf bytes.Buffer + timeStarted := time.Now() + + if outDir == "" { + log.Fatal("--out must be set") + } + + if siteDescriptionFile != "" { + siteDescriptionBytes, err := ioutil.ReadFile(siteDescriptionFile) + if err != nil { + log.Fatalf("failed to read site description file %s: %s", siteDescriptionFile, err) + } + siteDescription = string(siteDescriptionBytes) + } + + if siteDescription != "" { + markdown := goldmark.New( + goldmark.WithRendererOptions( + gmhtml.WithUnsafe(), + ), + goldmark.WithExtensions( + extension.NewLinkify(), + ), + ) + + buf.Reset() + err := markdown.Convert([]byte(siteDescription), &buf) + if err != nil { + log.Fatalf("failed to render site description markdown: %s", err) + } + siteDescription = buf.String() + } + + if verbose { + log.Println("Starting godoc...") + } + + cmd := exec.Command("godoc", fmt.Sprintf("-http=%s", listenAddress)) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Start() + if err != nil { + log.Fatalf("failed to execute godoc: %s", err) + } + + // Allow godoc to initialize + + time.Sleep(3 * time.Second) + + done := make(chan struct{}) + timeout := time.After(15 * time.Second) + + pkgs := flag.Args() + + var newPkgs []string + + for _, pkg := range pkgs { + if strings.TrimSpace(pkg) == "" { + continue + } + + buf.Reset() + + newPkgs = append(newPkgs, pkg) + + listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Dir }}`, pkg) + listCmd.Dir = os.TempDir() + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = &buf + + err = listCmd.Run() + if err != nil { + log.Fatalf("failed to list source directory of package %s: %s", pkg, err) + } + + pkgPath := strings.TrimSpace(buf.String()) + if pkgPath != "" { + err := filepath.Walk(pkgPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } else if !info.IsDir() { + return nil + } else if strings.HasPrefix(filepath.Base(path), ".") { + return filepath.SkipDir + } + + if len(path) > len(pkgPath) && strings.HasPrefix(path, pkgPath) { + newPkgs = append(newPkgs, pkg+path[len(pkgPath):]) + } + return nil + }) + if err != nil { + log.Fatalf("failed to walk source directory of package %s: %s", pkg, err) + } + } + buf.Reset() + } + pkgs = uniqueStrings(newPkgs) + + if len(pkgs) == 0 { + log.Fatal("failed to generate docs: provide the name of at least one package to generate documentation for") + } + + filterPkgs := pkgs + + for _, pkg := range pkgs { + subPkgs := strings.Split(pkg, "/") + for i := range subPkgs { + pkgs = append(pkgs, strings.Join(subPkgs[0:i+1], "/")) + } + } + pkgs = uniqueStrings(pkgs) + + sort.Slice(pkgs, func(i, j int) bool { + return strings.ToLower(pkgs[i]) < strings.ToLower(pkgs[j]) + }) + + if verbose { + log.Println("Copying docs...") + } + + go func() { + var ( + res *http.Response + err error + ) + for _, pkg := range pkgs { + // Rely on timeout to break loop + for { + res, err = http.Get(fmt.Sprintf("http://%s/pkg/%s/", listenAddress, pkg)) + if err == nil { + break + } + } + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + log.Fatalf("failed to get page of %s: %s", pkg, err) + } + + doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName)) + + updatePage(doc, basePath, siteName) + + localPkgPath := path.Join(outDir, pkg) + + err = os.MkdirAll(localPkgPath, 0755) + if err != nil { + log.Fatalf("failed to make directory %s: %s", localPkgPath, err) + } + + buf.Reset() + err = html.Render(&buf, doc.Nodes[0]) + if err != nil { + return + } + err = ioutil.WriteFile(path.Join(localPkgPath, "index.html"), buf.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write docs for %s: %s", pkg, err) + } + } + done <- struct{}{} + }() + + select { + case <-timeout: + log.Fatal("godoc failed to start in time") + case <-done: + } + + // Write source files + + if verbose { + log.Println("Copying sources...") + } + + err = os.MkdirAll(path.Join(outDir, "src"), 0755) + if err != nil { + log.Fatalf("failed to make directory lib: %s", err) + } + + for _, pkg := range filterPkgs { + tmpDir := os.TempDir() + // TODO Handle temp directory not existing + buf.Reset() + + listCmd := exec.Command("go", "list", "-find", "-f", `{{ join .GoFiles "\n" }}`, pkg) + listCmd.Dir = tmpDir + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = &buf + + err = listCmd.Run() + if err != nil { + //log.Fatalf("failed to list source files of package %s: %s", pkg, err) + continue // This is expected for packages without source files + } + + sourceFiles := strings.Split(buf.String(), "\n") + for _, sourceFile := range sourceFiles { + sourceFile = strings.TrimSpace(sourceFile) + if sourceFile == "" { + continue + } + + // Rely on timeout to break loop + res, err := http.Get(fmt.Sprintf("http://%s/src/%s/%s", listenAddress, pkg, sourceFile)) + if err != nil { + log.Fatalf("failed to get source file page %s for package %s: %s", sourceFile, pkg, err) + } + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + log.Fatalf("failed to load document from page for package %s: %s", pkg, err) + } + + doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName)) + + updatePage(doc, basePath, siteName) + + pkgSrcPath := path.Join(outDir, "src", pkg) + + err = os.MkdirAll(pkgSrcPath, 0755) + if err != nil { + log.Fatalf("failed to make directory %s: %s", pkgSrcPath, err) + } + + buf.Reset() + err = html.Render(&buf, doc.Nodes[0]) + if err != nil { + return + } + err = ioutil.WriteFile(path.Join(pkgSrcPath, sourceFile+".html"), buf.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write docs for %s: %s", pkg, err) + } + } + } + + // Write style.css + + if verbose { + log.Println("Copying style.css...") + } + + err = os.MkdirAll(path.Join(outDir, "lib"), 0755) + if err != nil { + log.Fatalf("failed to make directory lib: %s", err) + } + + res, err := http.Get(fmt.Sprintf("http://%s/lib/godoc/style.css", listenAddress)) + if err != nil { + log.Fatalf("failed to get syle.css: %s", err) + } + + content, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatalf("failed to get style.css: %s", err) + } + + content = append(content, []byte("\n"+additionalCSS+"\n")...) + + err = ioutil.WriteFile(path.Join(outDir, "lib", "style.css"), content, 0755) + if err != nil { + log.Fatalf("failed to write index: %s", err) + } + + // Write index + + if verbose { + log.Println("Writing index...") + } + + writeIndex(&buf, outDir, basePath, siteName, pkgs, filterPkgs) + + if verbose { + log.Printf("Generated documentation in %s", time.Since(timeStarted).Round(time.Second)) + } +} + +func uniqueStrings(strSlice []string) []string { + keys := make(map[string]bool) + var unique []string + for _, entry := range strSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + unique = append(unique, entry) + } + } + return unique +} diff --git a/page.go b/page.go new file mode 100644 index 0000000..5e60b8c --- /dev/null +++ b/page.go @@ -0,0 +1,243 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "strconv" + "strings" + "syscall" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" + + "github.com/PuerkitoBio/goquery" +) + +func topBar(basePath string, siteName string) string { + var index string + if linkIndex { + index = "index.html" + } + + return `` +} + +func updatePage(doc *goquery.Document, basePath string, siteName string) { + doc.Find("link").Remove() + doc.Find("script").Remove() + + linkTag := &html.Node{ + Type: html.ElementNode, + DataAtom: atom.Link, + Data: "link", + Attr: []html.Attribute{ + {Key: "type", Val: "text/css"}, + {Key: "rel", Val: "stylesheet"}, + {Key: "href", Val: basePath + "lib/style.css"}, + }, + } + + doc.Find("head").AppendNodes(linkTag) + + doc.Find("#topbar").First().SetHtml(topBar(basePath, siteName)) + + doc.Find("a").Each(func(_ int, selection *goquery.Selection) { + href := selection.AttrOr("href", "") + if strings.HasPrefix(href, "/src/") || strings.HasPrefix(href, "/pkg/") { + if strings.ContainsRune(path.Base(href), '.') { + queryPos := strings.IndexRune(href, '?') + if queryPos >= 0 { + href = href[0:queryPos] + ".html" + href[queryPos:] + } else { + hashPos := strings.IndexRune(href, '#') + if hashPos >= 0 { + href = href[0:hashPos] + ".html" + href[hashPos:] + } else { + href += ".html" + } + } + } else if linkIndex { + queryPos := strings.IndexRune(href, '?') + if queryPos >= 0 { + href = href[0:queryPos] + "/index.html" + href[queryPos:] + } else { + hashPos := strings.IndexRune(href, '#') + if hashPos >= 0 { + href = href[0:hashPos] + "/index.html" + href[hashPos:] + } else { + href += "/index.html" + } + } + } + + if strings.HasPrefix(href, "/pkg/") { + href = href[4:] + } + + selection.SetAttr("href", basePath+href[1:]) + } + }) + + doc.Find("div").Each(func(_ int, selection *goquery.Selection) { + if selection.HasClass("toggle") { + var summary string + var err error + selection.Find("div").Each(func(_ int, subSelection *goquery.Selection) { + if subSelection.HasClass("collapsed") { + summary, err = subSelection.Find("span.text").First().Html() + if err != nil { + summary = "Summary not available" + } + + subSelection.Remove() + } + }) + + selection.Find(".toggleButton").Remove() + + selection.PrependHtml(fmt.Sprintf("%s", summary)) + + selection.RemoveClass("toggle") + + selection.Nodes[0].Data = "details" + selection.Nodes[0].DataAtom = atom.Details + } + }) + + scriptTag := &html.Node{ + Type: html.ElementNode, + DataAtom: atom.Script, + Data: "script", + Attr: []html.Attribute{ + {Key: "type", Val: "text/javascript"}, + {Key: "src", Val: basePath + "lib/godoc-static.js"}, + }, + } + + doc.Find("body").AppendNodes(scriptTag) + + doc.Find("#footer").Last().Remove() +} + +func writeIndex(buf *bytes.Buffer, outDir string, basePath string, siteName string, pkgs []string, filterPkgs []string) { + var index string + if linkIndex { + index = "/index.html" + } + + var b bytes.Buffer + + b.WriteString(` + + + + + +` + siteName + ` + + + + +
+... +
+ +
` + topBar(basePath, siteName) + `
+
+
+`) + + if siteDescription != "" { + b.WriteString(siteDescription) + } + + b.WriteString(` +

+ Packages +

+
+ + + + + +`) + + var padding int + var lastPkg string + for _, pkg := range pkgs { + buf.Reset() + listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Doc }}`, pkg) + listCmd.Dir = os.TempDir() + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = buf + + _ = listCmd.Run() // Ignore error + + pkgLabel := pkg + if lastPkg != "" { + lastPkgSplit := strings.Split(lastPkg, "/") + pkgSplit := strings.Split(pkg, "/") + shared := 0 + for i := range pkgSplit { + if i < len(lastPkgSplit) && strings.ToLower(lastPkgSplit[i]) == strings.ToLower(pkgSplit[i]) { + shared++ + } + } + + padding = shared * 20 + pkgLabel = strings.Join(pkgSplit[shared:], "/") + } + lastPkg = pkg + + var linkPackage bool + for _, filterPkg := range filterPkgs { + if pkg == filterPkg { + linkPackage = true + break + } + } + b.WriteString(` + + + + +`) + } + b.WriteString(` +
NameSynopsis
`) + if !linkPackage { + b.WriteString(pkgLabel) + } else { + b.WriteString(`` + pkgLabel + ``) + } + b.WriteString(` + ` + buf.String() + ` +
+
+
+
+ + + +`) + + err := ioutil.WriteFile(path.Join(outDir, "index.html"), b.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write index: %s", err) + } +}