Initial commit

This commit is contained in:
Trevor Slocum 2020-02-05 09:28:02 -08:00
commit 59a830aadf
10 changed files with 711 additions and 0 deletions

5
.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
.idea/
dist/
vendor/
*.sh
godoc-static

26
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,26 @@
image: golang:latest
stages:
- validate
- build
fmt:
stage: validate
script:
- gofmt -l -s -e .
- exit $(gofmt -l -s -e . | wc -l)
vet:
stage: validate
script:
- go vet -composites=false ./...
test:
stage: validate
script:
- go test -race -v ./...
build:
stage: build
script:
- go build

2
CHANGELOG Normal file
View file

@ -0,0 +1,2 @@
0.1.0:
- Initial release

21
LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Trevor Slocum <trevor@rocketnine.space>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

38
README.md Normal file
View file

@ -0,0 +1,38 @@
# godoc-static
[![GoDoc](https://gitlab.com/tslocum/godoc-static/raw/master/badge.svg)](https://docs.rocketnine.space/gitlab.com/tslocum/godoc-static)
[![CI status](https://gitlab.com/tslocum/godoc-static/badges/master/pipeline.svg)](https://gitlab.com/tslocum/godoc-static/commits/master)
[![Donate](https://img.shields.io/liberapay/receives/rocketnine.space.svg?logo=liberapay)](https://liberapay.com/rocketnine.space)
Generate static Go documentation
## Demo
[Rocket Nine Labs Documentation](https://docs.rocketnine.space)
## Installation
Install `godoc-static`:
`go get gitlab.com/tslocum/godoc-static`
Also install `godoc`:
`go get golang.org/x/tools/cmd/godoc`
## Documentation
Execute `godoc-static` with the `-help` flag for more information.
### Usage examples
Generate documentation for `archive`, `fmt` and `net/http` targeting `https://docs.rocketnine.space`:
`godoc-static -base-path=/ -site-name="Rocket Nine Labs Documentation" -site-description="Welcome!" -out=/home/user/sites/docs archive fmt net/http`
Targeting `https://rocketnine.space/docs/`:
`godoc-static -base-path=/docs/ -site-name="Rocket Nine Labs Documentation" -site-description-file=/home/user/sitefiles/description.md -out=/home/user/sites/docs archive fmt net/http`
## Support
Please share issues/suggestions [here](https://gitlab.com/tslocum/godoc-static/issues).

1
badge.svg Executable file
View file

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="109" height="20"><linearGradient id="a" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><rect rx="3" width="109" height="20" fill="#555"/><rect rx="3" x="44" width="65" height="20" fill="#5272B4"/><path fill="#5272B4" d="M44 0h4v20h-4z"/><rect rx="3" width="109" height="20" fill="url(#a)"/><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="11"><text x="23" y="15" fill="#010101" fill-opacity=".3">godoc</text><text x="23" y="14">godoc</text><text x="75.5" y="15" fill="#010101" fill-opacity=".3">reference</text><text x="75.5" y="14">reference</text></g></svg>

After

Width:  |  Height:  |  Size: 733 B

9
go.mod Normal file
View file

@ -0,0 +1,9 @@
module gitlab.com/tslocum/godoc-static
go 1.13
require (
github.com/PuerkitoBio/goquery v1.5.1
github.com/yuin/goldmark v1.1.22
golang.org/x/net v0.0.0-20200202094626-16171245cfb2
)

12
go.sum Normal file
View file

@ -0,0 +1,12 @@
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/yuin/goldmark v1.1.22 h1:0e0f6Zee9SAQ5yOZGNMWaOxqVvcc/9/kUWu/Kl91Jk8=
github.com/yuin/goldmark v1.1.22/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

354
main.go Normal file
View file

@ -0,0 +1,354 @@
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
gmhtml "github.com/yuin/goldmark/renderer/html"
"golang.org/x/net/html"
)
const additionalCSS = `
details { margin-top: 20px; }
summary { margin-left: 20px; cursor: pointer; }
`
var (
listenAddress string
basePath string
siteName string
siteDescription string
siteDescriptionFile string
linkIndex bool
outDir string
verbose bool
)
func main() {
flag.StringVar(&listenAddress, "listen-address", "localhost:9001", "address for godoc to listen on while scraping pages")
flag.StringVar(&basePath, "base-path", "/", "site relative URL path with trailing slash")
flag.StringVar(&siteName, "site-name", "Documentation", "site name")
flag.StringVar(&siteDescription, "site-description", "", "site description (markdown-enabled)")
flag.StringVar(&siteDescriptionFile, "site-description-file", "", "path to markdown file containing site description")
flag.BoolVar(&linkIndex, "link-index", false, "set link targets to index.html instead of folder")
flag.StringVar(&outDir, "out", "", "site directory")
flag.BoolVar(&verbose, "verbose", false, "enable verbose logging")
flag.Parse()
var buf bytes.Buffer
timeStarted := time.Now()
if outDir == "" {
log.Fatal("--out must be set")
}
if siteDescriptionFile != "" {
siteDescriptionBytes, err := ioutil.ReadFile(siteDescriptionFile)
if err != nil {
log.Fatalf("failed to read site description file %s: %s", siteDescriptionFile, err)
}
siteDescription = string(siteDescriptionBytes)
}
if siteDescription != "" {
markdown := goldmark.New(
goldmark.WithRendererOptions(
gmhtml.WithUnsafe(),
),
goldmark.WithExtensions(
extension.NewLinkify(),
),
)
buf.Reset()
err := markdown.Convert([]byte(siteDescription), &buf)
if err != nil {
log.Fatalf("failed to render site description markdown: %s", err)
}
siteDescription = buf.String()
}
if verbose {
log.Println("Starting godoc...")
}
cmd := exec.Command("godoc", fmt.Sprintf("-http=%s", listenAddress))
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
log.Fatalf("failed to execute godoc: %s", err)
}
// Allow godoc to initialize
time.Sleep(3 * time.Second)
done := make(chan struct{})
timeout := time.After(15 * time.Second)
pkgs := flag.Args()
var newPkgs []string
for _, pkg := range pkgs {
if strings.TrimSpace(pkg) == "" {
continue
}
buf.Reset()
newPkgs = append(newPkgs, pkg)
listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Dir }}`, pkg)
listCmd.Dir = os.TempDir()
listCmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
listCmd.Stdout = &buf
err = listCmd.Run()
if err != nil {
log.Fatalf("failed to list source directory of package %s: %s", pkg, err)
}
pkgPath := strings.TrimSpace(buf.String())
if pkgPath != "" {
err := filepath.Walk(pkgPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
} else if !info.IsDir() {
return nil
} else if strings.HasPrefix(filepath.Base(path), ".") {
return filepath.SkipDir
}
if len(path) > len(pkgPath) && strings.HasPrefix(path, pkgPath) {
newPkgs = append(newPkgs, pkg+path[len(pkgPath):])
}
return nil
})
if err != nil {
log.Fatalf("failed to walk source directory of package %s: %s", pkg, err)
}
}
buf.Reset()
}
pkgs = uniqueStrings(newPkgs)
if len(pkgs) == 0 {
log.Fatal("failed to generate docs: provide the name of at least one package to generate documentation for")
}
filterPkgs := pkgs
for _, pkg := range pkgs {
subPkgs := strings.Split(pkg, "/")
for i := range subPkgs {
pkgs = append(pkgs, strings.Join(subPkgs[0:i+1], "/"))
}
}
pkgs = uniqueStrings(pkgs)
sort.Slice(pkgs, func(i, j int) bool {
return strings.ToLower(pkgs[i]) < strings.ToLower(pkgs[j])
})
if verbose {
log.Println("Copying docs...")
}
go func() {
var (
res *http.Response
err error
)
for _, pkg := range pkgs {
// Rely on timeout to break loop
for {
res, err = http.Get(fmt.Sprintf("http://%s/pkg/%s/", listenAddress, pkg))
if err == nil {
break
}
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatalf("failed to get page of %s: %s", pkg, err)
}
doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName))
updatePage(doc, basePath, siteName)
localPkgPath := path.Join(outDir, pkg)
err = os.MkdirAll(localPkgPath, 0755)
if err != nil {
log.Fatalf("failed to make directory %s: %s", localPkgPath, err)
}
buf.Reset()
err = html.Render(&buf, doc.Nodes[0])
if err != nil {
return
}
err = ioutil.WriteFile(path.Join(localPkgPath, "index.html"), buf.Bytes(), 0755)
if err != nil {
log.Fatalf("failed to write docs for %s: %s", pkg, err)
}
}
done <- struct{}{}
}()
select {
case <-timeout:
log.Fatal("godoc failed to start in time")
case <-done:
}
// Write source files
if verbose {
log.Println("Copying sources...")
}
err = os.MkdirAll(path.Join(outDir, "src"), 0755)
if err != nil {
log.Fatalf("failed to make directory lib: %s", err)
}
for _, pkg := range filterPkgs {
tmpDir := os.TempDir()
// TODO Handle temp directory not existing
buf.Reset()
listCmd := exec.Command("go", "list", "-find", "-f", `{{ join .GoFiles "\n" }}`, pkg)
listCmd.Dir = tmpDir
listCmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
listCmd.Stdout = &buf
err = listCmd.Run()
if err != nil {
//log.Fatalf("failed to list source files of package %s: %s", pkg, err)
continue // This is expected for packages without source files
}
sourceFiles := strings.Split(buf.String(), "\n")
for _, sourceFile := range sourceFiles {
sourceFile = strings.TrimSpace(sourceFile)
if sourceFile == "" {
continue
}
// Rely on timeout to break loop
res, err := http.Get(fmt.Sprintf("http://%s/src/%s/%s", listenAddress, pkg, sourceFile))
if err != nil {
log.Fatalf("failed to get source file page %s for package %s: %s", sourceFile, pkg, err)
}
// Load the HTML document
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatalf("failed to load document from page for package %s: %s", pkg, err)
}
doc.Find("title").First().SetHtml(fmt.Sprintf("%s - %s", path.Base(pkg), siteName))
updatePage(doc, basePath, siteName)
pkgSrcPath := path.Join(outDir, "src", pkg)
err = os.MkdirAll(pkgSrcPath, 0755)
if err != nil {
log.Fatalf("failed to make directory %s: %s", pkgSrcPath, err)
}
buf.Reset()
err = html.Render(&buf, doc.Nodes[0])
if err != nil {
return
}
err = ioutil.WriteFile(path.Join(pkgSrcPath, sourceFile+".html"), buf.Bytes(), 0755)
if err != nil {
log.Fatalf("failed to write docs for %s: %s", pkg, err)
}
}
}
// Write style.css
if verbose {
log.Println("Copying style.css...")
}
err = os.MkdirAll(path.Join(outDir, "lib"), 0755)
if err != nil {
log.Fatalf("failed to make directory lib: %s", err)
}
res, err := http.Get(fmt.Sprintf("http://%s/lib/godoc/style.css", listenAddress))
if err != nil {
log.Fatalf("failed to get syle.css: %s", err)
}
content, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
log.Fatalf("failed to get style.css: %s", err)
}
content = append(content, []byte("\n"+additionalCSS+"\n")...)
err = ioutil.WriteFile(path.Join(outDir, "lib", "style.css"), content, 0755)
if err != nil {
log.Fatalf("failed to write index: %s", err)
}
// Write index
if verbose {
log.Println("Writing index...")
}
writeIndex(&buf, outDir, basePath, siteName, pkgs, filterPkgs)
if verbose {
log.Printf("Generated documentation in %s", time.Since(timeStarted).Round(time.Second))
}
}
func uniqueStrings(strSlice []string) []string {
keys := make(map[string]bool)
var unique []string
for _, entry := range strSlice {
if _, value := keys[entry]; !value {
keys[entry] = true
unique = append(unique, entry)
}
}
return unique
}

243
page.go Normal file
View file

@ -0,0 +1,243 @@
package main
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"strconv"
"strings"
"syscall"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
"github.com/PuerkitoBio/goquery"
)
func topBar(basePath string, siteName string) string {
var index string
if linkIndex {
index = "index.html"
}
return `<div class="container">
<div class="top-heading" id="heading-wide"><a href="` + basePath + index + `">` + siteName + `</a></div>
<div class="top-heading" id="heading-narrow"><a href="` + basePath + index + `">` + siteName + `</a></div>
<!--<a href="#" id="menu-button"><span id="menu-button-arrow">&#9661;</span></a>-->
<div id="menu">
<a href="` + basePath + index + `" style="margin-right: 10px;">Packages</a>
</div>
</div>`
}
func updatePage(doc *goquery.Document, basePath string, siteName string) {
doc.Find("link").Remove()
doc.Find("script").Remove()
linkTag := &html.Node{
Type: html.ElementNode,
DataAtom: atom.Link,
Data: "link",
Attr: []html.Attribute{
{Key: "type", Val: "text/css"},
{Key: "rel", Val: "stylesheet"},
{Key: "href", Val: basePath + "lib/style.css"},
},
}
doc.Find("head").AppendNodes(linkTag)
doc.Find("#topbar").First().SetHtml(topBar(basePath, siteName))
doc.Find("a").Each(func(_ int, selection *goquery.Selection) {
href := selection.AttrOr("href", "")
if strings.HasPrefix(href, "/src/") || strings.HasPrefix(href, "/pkg/") {
if strings.ContainsRune(path.Base(href), '.') {
queryPos := strings.IndexRune(href, '?')
if queryPos >= 0 {
href = href[0:queryPos] + ".html" + href[queryPos:]
} else {
hashPos := strings.IndexRune(href, '#')
if hashPos >= 0 {
href = href[0:hashPos] + ".html" + href[hashPos:]
} else {
href += ".html"
}
}
} else if linkIndex {
queryPos := strings.IndexRune(href, '?')
if queryPos >= 0 {
href = href[0:queryPos] + "/index.html" + href[queryPos:]
} else {
hashPos := strings.IndexRune(href, '#')
if hashPos >= 0 {
href = href[0:hashPos] + "/index.html" + href[hashPos:]
} else {
href += "/index.html"
}
}
}
if strings.HasPrefix(href, "/pkg/") {
href = href[4:]
}
selection.SetAttr("href", basePath+href[1:])
}
})
doc.Find("div").Each(func(_ int, selection *goquery.Selection) {
if selection.HasClass("toggle") {
var summary string
var err error
selection.Find("div").Each(func(_ int, subSelection *goquery.Selection) {
if subSelection.HasClass("collapsed") {
summary, err = subSelection.Find("span.text").First().Html()
if err != nil {
summary = "Summary not available"
}
subSelection.Remove()
}
})
selection.Find(".toggleButton").Remove()
selection.PrependHtml(fmt.Sprintf("<summary>%s</summary>", summary))
selection.RemoveClass("toggle")
selection.Nodes[0].Data = "details"
selection.Nodes[0].DataAtom = atom.Details
}
})
scriptTag := &html.Node{
Type: html.ElementNode,
DataAtom: atom.Script,
Data: "script",
Attr: []html.Attribute{
{Key: "type", Val: "text/javascript"},
{Key: "src", Val: basePath + "lib/godoc-static.js"},
},
}
doc.Find("body").AppendNodes(scriptTag)
doc.Find("#footer").Last().Remove()
}
func writeIndex(buf *bytes.Buffer, outDir string, basePath string, siteName string, pkgs []string, filterPkgs []string) {
var index string
if linkIndex {
index = "/index.html"
}
var b bytes.Buffer
b.WriteString(`<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#375EAB">
<title>` + siteName + `</title>
<link type="text/css" rel="stylesheet" href="` + basePath + `lib/style.css">
</head>
<body>
<div id="lowframe" style="position: fixed; bottom: 0; left: 0; height: 0; width: 100%; border-top: thin solid grey; background-color: white; overflow: auto;">
...
</div><!-- #lowframe -->
<div id="topbar" class="wide">` + topBar(basePath, siteName) + `</div>
<div id="page" class="wide">
<div class="container">
`)
if siteDescription != "" {
b.WriteString(siteDescription)
}
b.WriteString(`
<h1>
Packages
</h1>
<div class="pkg-dir">
<table>
<tr>
<th class="pkg-name">Name</th>
<th class="pkg-synopsis">Synopsis</th>
</tr>
`)
var padding int
var lastPkg string
for _, pkg := range pkgs {
buf.Reset()
listCmd := exec.Command("go", "list", "-find", "-f", `{{ .Doc }}`, pkg)
listCmd.Dir = os.TempDir()
listCmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
listCmd.Stdout = buf
_ = listCmd.Run() // Ignore error
pkgLabel := pkg
if lastPkg != "" {
lastPkgSplit := strings.Split(lastPkg, "/")
pkgSplit := strings.Split(pkg, "/")
shared := 0
for i := range pkgSplit {
if i < len(lastPkgSplit) && strings.ToLower(lastPkgSplit[i]) == strings.ToLower(pkgSplit[i]) {
shared++
}
}
padding = shared * 20
pkgLabel = strings.Join(pkgSplit[shared:], "/")
}
lastPkg = pkg
var linkPackage bool
for _, filterPkg := range filterPkgs {
if pkg == filterPkg {
linkPackage = true
break
}
}
b.WriteString(`
<tr>
<td class="pkg-name" style="padding-left: ` + strconv.Itoa(padding) + `px;">`)
if !linkPackage {
b.WriteString(pkgLabel)
} else {
b.WriteString(`<a href="` + pkg + index + `">` + pkgLabel + `</a>`)
}
b.WriteString(`</td>
<td class="pkg-synopsis">
` + buf.String() + `
</td>
</tr>
`)
}
b.WriteString(`
</table>
</div>
</div>
</div>
<script type="text/javascript" src="` + basePath + `lib/godoc-static.js"></script>
</body>
</html>
`)
err := ioutil.WriteFile(path.Join(outDir, "index.html"), b.Bytes(), 0755)
if err != nil {
log.Fatalf("failed to write index: %s", err)
}
}