From 59a830aadfaea7dd977db018d88ccf7e006b82a2 Mon Sep 17 00:00:00 2001 From: Trevor Slocum Date: Wed, 5 Feb 2020 09:28:02 -0800 Subject: [PATCH] Initial commit --- .gitignore | 5 + .gitlab-ci.yml | 26 ++++ CHANGELOG | 2 + LICENSE | 21 +++ README.md | 38 ++++++ badge.svg | 1 + go.mod | 9 ++ go.sum | 12 ++ main.go | 354 +++++++++++++++++++++++++++++++++++++++++++++++++ page.go | 243 +++++++++++++++++++++++++++++++++ 10 files changed, 711 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 CHANGELOG create mode 100644 LICENSE create mode 100644 README.md create mode 100755 badge.svg create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 page.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95b815f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea/ +dist/ +vendor/ +*.sh +godoc-static diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..1aab122 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,26 @@ +image: golang:latest + +stages: + - validate + - build + +fmt: + stage: validate + script: + - gofmt -l -s -e . + - exit $(gofmt -l -s -e . | wc -l) + +vet: + stage: validate + script: + - go vet -composites=false ./... + +test: + stage: validate + script: + - go test -race -v ./... + +build: + stage: build + script: + - go build diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..8da6a6f --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,2 @@ +0.1.0: +- Initial release diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7c20f89 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Trevor Slocum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0bc4091 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# godoc-static +[![GoDoc](https://gitlab.com/tslocum/godoc-static/raw/master/badge.svg)](https://docs.rocketnine.space/gitlab.com/tslocum/godoc-static) +[![CI status](https://gitlab.com/tslocum/godoc-static/badges/master/pipeline.svg)](https://gitlab.com/tslocum/godoc-static/commits/master) +[![Donate](https://img.shields.io/liberapay/receives/rocketnine.space.svg?logo=liberapay)](https://liberapay.com/rocketnine.space) + +Generate static Go documentation + +## Demo + +[Rocket Nine Labs Documentation](https://docs.rocketnine.space) + +## Installation + +Install `godoc-static`: + +`go get gitlab.com/tslocum/godoc-static` + +Also install `godoc`: + +`go get golang.org/x/tools/cmd/godoc` + +## Documentation + +Execute `godoc-static` with the `-help` flag for more information. + +### Usage examples + +Generate documentation for `archive`, `fmt` and `net/http` targeting `https://docs.rocketnine.space`: + +`godoc-static -base-path=/ -site-name="Rocket Nine Labs Documentation" -site-description="Welcome!" -out=/home/user/sites/docs archive fmt net/http` + +Targeting `https://rocketnine.space/docs/`: + +`godoc-static -base-path=/docs/ -site-name="Rocket Nine Labs Documentation" -site-description-file=/home/user/sitefiles/description.md -out=/home/user/sites/docs archive fmt net/http` + +## Support + +Please share issues/suggestions [here](https://gitlab.com/tslocum/godoc-static/issues). diff --git a/badge.svg b/badge.svg new file mode 100755 index 0000000..3952e41 --- /dev/null +++ b/badge.svg @@ -0,0 +1 @@ +godocgodocreferencereference \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1c55b34 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module gitlab.com/tslocum/godoc-static + +go 1.13 + +require ( + github.com/PuerkitoBio/goquery v1.5.1 + github.com/yuin/goldmark v1.1.22 + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1f40cb4 --- /dev/null +++ b/go.sum @@ -0,0 +1,12 @@ +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/yuin/goldmark v1.1.22 h1:0e0f6Zee9SAQ5yOZGNMWaOxqVvcc/9/kUWu/Kl91Jk8= +github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..1b1e8a9 --- /dev/null +++ b/main.go @@ -0,0 +1,354 @@ +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" + gmhtml "github.com/yuin/goldmark/renderer/html" + "golang.org/x/net/html" +) + +const additionalCSS = ` +details { margin-top: 20px; } +summary { margin-left: 20px; cursor: pointer; } +` + +var ( + listenAddress string + basePath string + siteName string + siteDescription string + siteDescriptionFile string + linkIndex bool + outDir string + verbose bool +) + +func main() { + flag.StringVar(&listenAddress, "listen-address", "localhost:9001", "address for godoc to listen on while scraping pages") + flag.StringVar(&basePath, "base-path", "/", "site relative URL path with trailing slash") + flag.StringVar(&siteName, "site-name", "Documentation", "site name") + flag.StringVar(&siteDescription, "site-description", "", "site description (markdown-enabled)") + flag.StringVar(&siteDescriptionFile, "site-description-file", "", "path to markdown file containing site description") + flag.BoolVar(&linkIndex, "link-index", false, "set link targets to index.html instead of folder") + flag.StringVar(&outDir, "out", "", "site directory") + flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") + flag.Parse() + + var buf bytes.Buffer + timeStarted := time.Now() + + if outDir == "" { + log.Fatal("--out must be set") + } + + if siteDescriptionFile != "" { + siteDescriptionBytes, err := ioutil.ReadFile(siteDescriptionFile) + if err != nil { + log.Fatalf("failed to read site description file %s: %s", siteDescriptionFile, err) + } + siteDescription = string(siteDescriptionBytes) + } + + if siteDescription != "" { + markdown := goldmark.New( + goldmark.WithRendererOptions( + gmhtml.WithUnsafe(), + ), + goldmark.WithExtensions( + extension.NewLinkify(), + ), + ) + + buf.Reset() + err := markdown.Convert([]byte(siteDescription), &buf) + if err != nil { + log.Fatalf("failed to render site description markdown: %s", err) + } + siteDescription = buf.String() + } + + if verbose { + log.Println("Starting godoc...") + } + + cmd := exec.Command("godoc", fmt.Sprintf("-http=%s", listenAddress)) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Start() + if err != nil { + log.Fatalf("failed to execute godoc: %s", err) + } + + // Allow godoc to initialize + + time.Sleep(3 * time.Second) + + done := make(chan struct{}) + timeout := time.After(15 * time.Second) + + pkgs := flag.Args() + + var newPkgs []string + + for _, pkg := range pkgs { + if strings.TrimSpace(pkg) == "" { + continue + } + + buf.Reset() + + newPkgs = append(newPkgs, pkg) + + listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Dir }}`, pkg) + listCmd.Dir = os.TempDir() + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = &buf + + err = listCmd.Run() + if err != nil { + log.Fatalf("failed to list source directory of package %s: %s", pkg, err) + } + + pkgPath := strings.TrimSpace(buf.String()) + if pkgPath != "" { + err := filepath.Walk(pkgPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } else if !info.IsDir() { + return nil + } else if strings.HasPrefix(filepath.Base(path), ".") { + return filepath.SkipDir + } + + if len(path) > len(pkgPath) && strings.HasPrefix(path, pkgPath) { + newPkgs = append(newPkgs, pkg+path[len(pkgPath):]) + } + return nil + }) + if err != nil { + log.Fatalf("failed to walk source directory of package %s: %s", pkg, err) + } + } + buf.Reset() + } + pkgs = uniqueStrings(newPkgs) + + if len(pkgs) == 0 { + log.Fatal("failed to generate docs: provide the name of at least one package to generate documentation for") + } + + filterPkgs := pkgs + + for _, pkg := range pkgs { + subPkgs := strings.Split(pkg, "/") + for i := range subPkgs { + pkgs = append(pkgs, strings.Join(subPkgs[0:i+1], "/")) + } + } + pkgs = uniqueStrings(pkgs) + + sort.Slice(pkgs, func(i, j int) bool { + return strings.ToLower(pkgs[i]) < strings.ToLower(pkgs[j]) + }) + + if verbose { + log.Println("Copying docs...") + } + + go func() { + var ( + res *http.Response + err error + ) + for _, pkg := range pkgs { + // Rely on timeout to break loop + for { + res, err = http.Get(fmt.Sprintf("http://%s/pkg/%s/", listenAddress, pkg)) + if err == nil { + break + } + } + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + log.Fatalf("failed to get page of %s: %s", pkg, err) + } + + doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName)) + + updatePage(doc, basePath, siteName) + + localPkgPath := path.Join(outDir, pkg) + + err = os.MkdirAll(localPkgPath, 0755) + if err != nil { + log.Fatalf("failed to make directory %s: %s", localPkgPath, err) + } + + buf.Reset() + err = html.Render(&buf, doc.Nodes[0]) + if err != nil { + return + } + err = ioutil.WriteFile(path.Join(localPkgPath, "index.html"), buf.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write docs for %s: %s", pkg, err) + } + } + done <- struct{}{} + }() + + select { + case <-timeout: + log.Fatal("godoc failed to start in time") + case <-done: + } + + // Write source files + + if verbose { + log.Println("Copying sources...") + } + + err = os.MkdirAll(path.Join(outDir, "src"), 0755) + if err != nil { + log.Fatalf("failed to make directory lib: %s", err) + } + + for _, pkg := range filterPkgs { + tmpDir := os.TempDir() + // TODO Handle temp directory not existing + buf.Reset() + + listCmd := exec.Command("go", "list", "-find", "-f", `{{ join .GoFiles "\n" }}`, pkg) + listCmd.Dir = tmpDir + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = &buf + + err = listCmd.Run() + if err != nil { + //log.Fatalf("failed to list source files of package %s: %s", pkg, err) + continue // This is expected for packages without source files + } + + sourceFiles := strings.Split(buf.String(), "\n") + for _, sourceFile := range sourceFiles { + sourceFile = strings.TrimSpace(sourceFile) + if sourceFile == "" { + continue + } + + // Rely on timeout to break loop + res, err := http.Get(fmt.Sprintf("http://%s/src/%s/%s", listenAddress, pkg, sourceFile)) + if err != nil { + log.Fatalf("failed to get source file page %s for package %s: %s", sourceFile, pkg, err) + } + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + log.Fatalf("failed to load document from page for package %s: %s", pkg, err) + } + + doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName)) + + updatePage(doc, basePath, siteName) + + pkgSrcPath := path.Join(outDir, "src", pkg) + + err = os.MkdirAll(pkgSrcPath, 0755) + if err != nil { + log.Fatalf("failed to make directory %s: %s", pkgSrcPath, err) + } + + buf.Reset() + err = html.Render(&buf, doc.Nodes[0]) + if err != nil { + return + } + err = ioutil.WriteFile(path.Join(pkgSrcPath, sourceFile+".html"), buf.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write docs for %s: %s", pkg, err) + } + } + } + + // Write style.css + + if verbose { + log.Println("Copying style.css...") + } + + err = os.MkdirAll(path.Join(outDir, "lib"), 0755) + if err != nil { + log.Fatalf("failed to make directory lib: %s", err) + } + + res, err := http.Get(fmt.Sprintf("http://%s/lib/godoc/style.css", listenAddress)) + if err != nil { + log.Fatalf("failed to get syle.css: %s", err) + } + + content, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatalf("failed to get style.css: %s", err) + } + + content = append(content, []byte("\n"+additionalCSS+"\n")...) + + err = ioutil.WriteFile(path.Join(outDir, "lib", "style.css"), content, 0755) + if err != nil { + log.Fatalf("failed to write index: %s", err) + } + + // Write index + + if verbose { + log.Println("Writing index...") + } + + writeIndex(&buf, outDir, basePath, siteName, pkgs, filterPkgs) + + if verbose { + log.Printf("Generated documentation in %s", time.Since(timeStarted).Round(time.Second)) + } +} + +func uniqueStrings(strSlice []string) []string { + keys := make(map[string]bool) + var unique []string + for _, entry := range strSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + unique = append(unique, entry) + } + } + return unique +} diff --git a/page.go b/page.go new file mode 100644 index 0000000..5e60b8c --- /dev/null +++ b/page.go @@ -0,0 +1,243 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path" + "strconv" + "strings" + "syscall" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" + + "github.com/PuerkitoBio/goquery" +) + +func topBar(basePath string, siteName string) string { + var index string + if linkIndex { + index = "index.html" + } + + return `
+ + + + +
` +} + +func updatePage(doc *goquery.Document, basePath string, siteName string) { + doc.Find("link").Remove() + doc.Find("script").Remove() + + linkTag := &html.Node{ + Type: html.ElementNode, + DataAtom: atom.Link, + Data: "link", + Attr: []html.Attribute{ + {Key: "type", Val: "text/css"}, + {Key: "rel", Val: "stylesheet"}, + {Key: "href", Val: basePath + "lib/style.css"}, + }, + } + + doc.Find("head").AppendNodes(linkTag) + + doc.Find("#topbar").First().SetHtml(topBar(basePath, siteName)) + + doc.Find("a").Each(func(_ int, selection *goquery.Selection) { + href := selection.AttrOr("href", "") + if strings.HasPrefix(href, "/src/") || strings.HasPrefix(href, "/pkg/") { + if strings.ContainsRune(path.Base(href), '.') { + queryPos := strings.IndexRune(href, '?') + if queryPos >= 0 { + href = href[0:queryPos] + ".html" + href[queryPos:] + } else { + hashPos := strings.IndexRune(href, '#') + if hashPos >= 0 { + href = href[0:hashPos] + ".html" + href[hashPos:] + } else { + href += ".html" + } + } + } else if linkIndex { + queryPos := strings.IndexRune(href, '?') + if queryPos >= 0 { + href = href[0:queryPos] + "/index.html" + href[queryPos:] + } else { + hashPos := strings.IndexRune(href, '#') + if hashPos >= 0 { + href = href[0:hashPos] + "/index.html" + href[hashPos:] + } else { + href += "/index.html" + } + } + } + + if strings.HasPrefix(href, "/pkg/") { + href = href[4:] + } + + selection.SetAttr("href", basePath+href[1:]) + } + }) + + doc.Find("div").Each(func(_ int, selection *goquery.Selection) { + if selection.HasClass("toggle") { + var summary string + var err error + selection.Find("div").Each(func(_ int, subSelection *goquery.Selection) { + if subSelection.HasClass("collapsed") { + summary, err = subSelection.Find("span.text").First().Html() + if err != nil { + summary = "Summary not available" + } + + subSelection.Remove() + } + }) + + selection.Find(".toggleButton").Remove() + + selection.PrependHtml(fmt.Sprintf("%s", summary)) + + selection.RemoveClass("toggle") + + selection.Nodes[0].Data = "details" + selection.Nodes[0].DataAtom = atom.Details + } + }) + + scriptTag := &html.Node{ + Type: html.ElementNode, + DataAtom: atom.Script, + Data: "script", + Attr: []html.Attribute{ + {Key: "type", Val: "text/javascript"}, + {Key: "src", Val: basePath + "lib/godoc-static.js"}, + }, + } + + doc.Find("body").AppendNodes(scriptTag) + + doc.Find("#footer").Last().Remove() +} + +func writeIndex(buf *bytes.Buffer, outDir string, basePath string, siteName string, pkgs []string, filterPkgs []string) { + var index string + if linkIndex { + index = "/index.html" + } + + var b bytes.Buffer + + b.WriteString(` + + + + + +` + siteName + ` + + + + +
+... +
+ +
` + topBar(basePath, siteName) + `
+
+
+`) + + if siteDescription != "" { + b.WriteString(siteDescription) + } + + b.WriteString(` +

+ Packages +

+
+ + + + + +`) + + var padding int + var lastPkg string + for _, pkg := range pkgs { + buf.Reset() + listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Doc }}`, pkg) + listCmd.Dir = os.TempDir() + listCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } + listCmd.Stdout = buf + + _ = listCmd.Run() // Ignore error + + pkgLabel := pkg + if lastPkg != "" { + lastPkgSplit := strings.Split(lastPkg, "/") + pkgSplit := strings.Split(pkg, "/") + shared := 0 + for i := range pkgSplit { + if i < len(lastPkgSplit) && strings.ToLower(lastPkgSplit[i]) == strings.ToLower(pkgSplit[i]) { + shared++ + } + } + + padding = shared * 20 + pkgLabel = strings.Join(pkgSplit[shared:], "/") + } + lastPkg = pkg + + var linkPackage bool + for _, filterPkg := range filterPkgs { + if pkg == filterPkg { + linkPackage = true + break + } + } + b.WriteString(` + + + + +`) + } + b.WriteString(` +
NameSynopsis
`) + if !linkPackage { + b.WriteString(pkgLabel) + } else { + b.WriteString(`` + pkgLabel + ``) + } + b.WriteString(` + ` + buf.String() + ` +
+
+
+
+ + + +`) + + err := ioutil.WriteFile(path.Join(outDir, "index.html"), b.Bytes(), 0755) + if err != nil { + log.Fatalf("failed to write index: %s", err) + } +}