Rewrite as library
This commit is contained in:
parent
845655a1ac
commit
1e1f83ea18
12 changed files with 542 additions and 411 deletions
|
|
@ -18,10 +18,10 @@ tasks:
|
|||
go mod download
|
||||
- test: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
go test
|
||||
go test -v -cover ./...
|
||||
- build: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
cd $PROJECT_DIR/$PROJECT_NAME/cmd/$PROJECT_NAME
|
||||
go build
|
||||
- run: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
cd $PROJECT_DIR/$PROJECT_NAME/cmd/$PROJECT_NAME
|
||||
./$PROJECT_NAME --help
|
||||
|
|
|
|||
|
|
@ -18,10 +18,10 @@ tasks:
|
|||
go mod download
|
||||
- test: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
go test
|
||||
go test -v -cover ./...
|
||||
- build: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
cd $PROJECT_DIR/$PROJECT_NAME/cmd/$PROJECT_NAME
|
||||
go build
|
||||
- run: |
|
||||
cd $PROJECT_DIR/$PROJECT_NAME
|
||||
cd $PROJECT_DIR/$PROJECT_NAME/cmd/$PROJECT_NAME
|
||||
./$PROJECT_NAME --help
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -1,5 +1,5 @@
|
|||
.idea/
|
||||
dist/
|
||||
*.sh
|
||||
gophast
|
||||
cmd/gophast/gophast
|
||||
vendor/
|
||||
|
|
|
|||
101
cmd/gophast/main.go
Normal file
101
cmd/gophast/main.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
|
||||
"git.sr.ht/~tslocum/gophast/pkg/gophast"
|
||||
"git.sr.ht/~tslocum/gophast/pkg/progress"
|
||||
isatty "github.com/mattn/go-isatty"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime))
|
||||
|
||||
app := cli.NewApp()
|
||||
app.HideVersion = true
|
||||
|
||||
app.Usage = "Download accelerator"
|
||||
|
||||
cli.AppHelpTemplate = `NAME:
|
||||
{{.Name}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} [OPTIONS] URL
|
||||
|
||||
OPTIONS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}
|
||||
`
|
||||
|
||||
app.Flags = []cli.Flag{
|
||||
cli.StringFlag{Name: "dir, d", Usage: "directory to save in (default: current directory)", EnvVar: "GOPHAST_DIR"},
|
||||
cli.StringFlag{Name: "out, o", Usage: "file name to save as (default: from URL)", EnvVar: "GOPHAST_OUT"},
|
||||
cli.Int64Flag{Name: "min-split-size, k", Usage: "minimum byte range size", EnvVar: "GOPHAST_SPLIT_SIZE", Value: gophast.DefaultMinSplitSize},
|
||||
cli.Int64Flag{Name: "split-servers, s", Usage: "split download using multiple URLs", EnvVar: "GOPHAST_SPLIT_SERVERS", Value: 1},
|
||||
cli.Int64Flag{Name: "split-conns, x", Usage: "split download using multiple connections", EnvVar: "GOPHAST_SPLIT_CONNECTIONS", Value: gophast.DefaultMaxConnections},
|
||||
cli.Int64Flag{Name: "downloads, j", Usage: "number of files to download concurrently", EnvVar: "GOPHAST_DOWNLOADS", Value: 5},
|
||||
cli.BoolFlag{Name: "no-progress, p", Usage: "disable progress bars", EnvVar: "GOPHAST_NO_PROGRESS"},
|
||||
cli.BoolFlag{Name: "no-resume, r", Usage: "discard existing data before downloading", EnvVar: "GOPHAST_NO_RESUME"},
|
||||
cli.BoolFlag{Name: "quiet, q", Usage: "print error messages only", EnvVar: "GOPHAST_QUIET"},
|
||||
cli.BoolFlag{Name: "verbose, v", Usage: "print verbose messages (disables progress bars)", EnvVar: "GOPHAST_VERBOSE"},
|
||||
}
|
||||
sort.Sort(cli.FlagsByName(app.Flags))
|
||||
|
||||
app.Action = GoPhast
|
||||
|
||||
err := app.Run(os.Args)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func GoPhast(c *cli.Context) error {
|
||||
if !c.Bool("no-progress") && (isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) {
|
||||
gophast.C.ProgressLevel = progress.Dynamic
|
||||
}
|
||||
|
||||
gophast.C.MaxLogLevel = gophast.LogStandard
|
||||
if c.Bool("quiet") && c.Bool("verbose") {
|
||||
panic("--quiet and --verbose options are mutually exclusive")
|
||||
} else if c.Bool("quiet") {
|
||||
gophast.C.MaxLogLevel = gophast.LogNone
|
||||
} else if c.Bool("verbose") {
|
||||
gophast.C.MaxLogLevel = gophast.LogVerbose
|
||||
gophast.C.ProgressLevel = progress.Static
|
||||
}
|
||||
|
||||
gophast.C.MinSplitSize = c.Int64("min-split-size")
|
||||
gophast.C.MaxConnections = c.Int64("split-conns")
|
||||
|
||||
downloadDir := c.String("dir")
|
||||
if downloadDir != "" {
|
||||
gophast.C.DownloadDir = downloadDir
|
||||
}
|
||||
gophast.C.DownloadName = c.String("out")
|
||||
gophast.C.NoResume = c.Bool("no-resume")
|
||||
|
||||
var postData []byte
|
||||
|
||||
if gophast.C.MinSplitSize < 0 {
|
||||
return errors.New("invalid minimum byte range request size (--min-split-size)")
|
||||
}
|
||||
|
||||
downloadURL := c.Args().First()
|
||||
if downloadURL == "" {
|
||||
return c.App.Command("help").Run(c)
|
||||
}
|
||||
|
||||
gophast.LogMessage(gophast.LogStandard, "Fetching metadata...")
|
||||
downloadName, downloadSize := gophast.Metadata(downloadURL, postData)
|
||||
|
||||
if !gophast.Download(downloadURL, postData, gophast.DownloadPath(downloadName), downloadSize) {
|
||||
log.Fatal("failed to download file")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
3
go.mod
3
go.mod
|
|
@ -6,7 +6,4 @@ require (
|
|||
github.com/mattn/go-isatty v0.0.7
|
||||
github.com/urfave/cli v1.20.0
|
||||
github.com/vbauerster/mpb/v4 v4.7.0
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
golang.org/x/tools v0.0.0-20190501045030-23463209683d // indirect
|
||||
)
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -9,16 +9,10 @@ github.com/vbauerster/mpb/v4 v4.7.0/go.mod h1:ugxYn2kSUrY10WK5CWDUZvQxjdwKFN9K3J
|
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
|
||||
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190501045030-23463209683d/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
|
|
|
|||
395
main.go
395
main.go
|
|
@ -1,395 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
isatty "github.com/mattn/go-isatty"
|
||||
"github.com/urfave/cli"
|
||||
mpb "github.com/vbauerster/mpb/v4"
|
||||
"github.com/vbauerster/mpb/v4/decor"
|
||||
)
|
||||
|
||||
const DefaultMinimumSplitSize = 10 * 1024 * 1024 // 10 megabytes
|
||||
|
||||
type LogLevel int
|
||||
|
||||
const (
|
||||
LogNone = LogLevel(0)
|
||||
LogStandard = LogLevel(1)
|
||||
LogVerbose = LogLevel(2)
|
||||
)
|
||||
|
||||
type ProgressLevel int
|
||||
|
||||
const (
|
||||
ProgressStatic = ProgressLevel(0)
|
||||
ProgressDynamic = ProgressLevel(1)
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MaxLogLevel LogLevel
|
||||
ProgressLevel ProgressLevel
|
||||
}
|
||||
|
||||
type ProgressReport struct {
|
||||
wrote int
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
var (
|
||||
config = new(Config)
|
||||
wg = new(sync.WaitGroup)
|
||||
progress *mpb.Progress
|
||||
progressBars []*mpb.Bar
|
||||
progressTaskWidth int
|
||||
progressTaskFormat string
|
||||
progressWrote = make(chan *ProgressReport, 10)
|
||||
)
|
||||
|
||||
type OffsetWriter struct {
|
||||
io.WriterAt
|
||||
offset int64
|
||||
|
||||
bar *mpb.Bar
|
||||
lastUpdate time.Time
|
||||
}
|
||||
|
||||
func (dst *OffsetWriter) Write(b []byte) (n int, err error) {
|
||||
n, err = dst.WriteAt(b, dst.offset)
|
||||
dst.offset += int64(n)
|
||||
|
||||
if dst.bar != nil {
|
||||
dst.bar.IncrBy(n, time.Since(dst.lastUpdate))
|
||||
progressWrote <- &ProgressReport{n, time.Since(dst.lastUpdate)}
|
||||
dst.lastUpdate = time.Now()
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func CheckError(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func formatBytes(bytes int64) string {
|
||||
if bytes == 0 {
|
||||
return "0 B"
|
||||
}
|
||||
|
||||
k := float64(1024)
|
||||
sizes := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
|
||||
|
||||
i := math.Floor(math.Log(float64(bytes)) / math.Log(k))
|
||||
|
||||
format := "%.1f"
|
||||
if i == 0 {
|
||||
format = "%f"
|
||||
}
|
||||
|
||||
return fmt.Sprintf(format+" %s", float64(bytes)/math.Pow(k, i), sizes[int(i)])
|
||||
}
|
||||
|
||||
func request(method string, url string, postData []byte, rangeStart int64, rangeEnd int64) (*http.Response, error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest(method, url, bytes.NewReader(postData))
|
||||
CheckError(err)
|
||||
|
||||
req.Header.Add("User-Agent", "gophast https://git.sr.ht/~tslocum/gophast")
|
||||
if rangeEnd > 0 {
|
||||
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd))
|
||||
}
|
||||
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func metadata(url string, postData []byte) (string, int64) {
|
||||
name := path.Base(url)
|
||||
|
||||
resp, err := request("HEAD", url, postData, 0, 0)
|
||||
CheckError(err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 404 {
|
||||
log.Fatal("404: not found")
|
||||
} else if resp.StatusCode != 200 {
|
||||
log.Fatalf("server responded with status %d (expected 200 OK)", resp.StatusCode)
|
||||
}
|
||||
|
||||
if resp.ContentLength < 0 {
|
||||
panic("server did not send Content-Length")
|
||||
}
|
||||
|
||||
if !strings.Contains(resp.Header.Get("Accept-Ranges"), "bytes") {
|
||||
panic("server does not support Range")
|
||||
}
|
||||
|
||||
// TODO: Handle redirect, update name
|
||||
|
||||
return name, resp.ContentLength
|
||||
}
|
||||
|
||||
func download(url string, postData []byte, rangeStart int64, rangeEnd int64, file *os.File, id int64, maxid int64, bar *mpb.Bar) {
|
||||
var (
|
||||
wrote int64
|
||||
)
|
||||
defer wg.Done()
|
||||
start := time.Now()
|
||||
|
||||
rangeSize := rangeEnd - rangeStart + 1
|
||||
|
||||
resp, err := request("GET", url, postData, rangeStart, rangeEnd)
|
||||
CheckError(err)
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
panic(fmt.Sprintf("invalid response code: %d (expected: 206 Partial Content)", resp.StatusCode))
|
||||
}
|
||||
|
||||
logMessagef(LogVerbose, "Starting range %d/%d (%d-%d)", id, maxid, rangeStart, rangeEnd)
|
||||
|
||||
ow := &OffsetWriter{WriterAt: file, offset: rangeStart, bar: bar, lastUpdate: time.Now()}
|
||||
|
||||
wrote, err = io.Copy(ow, resp.Body)
|
||||
CheckError(err)
|
||||
if wrote != rangeSize {
|
||||
_ = resp.Body.Close()
|
||||
panic(fmt.Sprintf("received %d bytes of expected %d", wrote, rangeSize))
|
||||
}
|
||||
|
||||
err = resp.Body.Close()
|
||||
CheckError(err)
|
||||
|
||||
logMessagef(LogVerbose, "Finished range %d/%d in %s (%s @ %s/s)", id, maxid, time.Since(start).Truncate(time.Second), formatBytes(rangeSize), formatBytes(int64(float64(float64(rangeSize))/time.Since(start).Seconds())))
|
||||
}
|
||||
|
||||
func logMessage(level LogLevel, message string) {
|
||||
if level > config.MaxLogLevel {
|
||||
return
|
||||
}
|
||||
|
||||
log.Println(message)
|
||||
}
|
||||
|
||||
func logMessagef(level LogLevel, format string, a ...interface{}) {
|
||||
logMessage(level, fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
func handleProgress(bar *mpb.Bar) {
|
||||
var report *ProgressReport
|
||||
for {
|
||||
report = <-progressWrote
|
||||
bar.IncrBy(report.wrote, report.duration)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime))
|
||||
|
||||
app := cli.NewApp()
|
||||
app.HideVersion = true
|
||||
|
||||
app.Usage = "Download accelerator"
|
||||
|
||||
cli.AppHelpTemplate = `NAME:
|
||||
{{.Name}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} [OPTIONS] URL
|
||||
|
||||
OPTIONS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}
|
||||
`
|
||||
|
||||
app.Flags = []cli.Flag{
|
||||
cli.StringFlag{Name: "dir, d", Usage: "directory to save in (default: current directory)", EnvVar: "GOPHAST_DIR"},
|
||||
cli.StringFlag{Name: "out, o", Usage: "file name to save as (default: from URL)", EnvVar: "GOPHAST_OUT"},
|
||||
cli.Int64Flag{Name: "split-size, k", Usage: "minimum byte range size", EnvVar: "GOPHAST_SPLIT_SIZE", Value: DefaultMinimumSplitSize},
|
||||
cli.Int64Flag{Name: "split-servers, s", Usage: "split download using multiple URLs", EnvVar: "GOPHAST_SPLIT_SERVERS", Value: 1},
|
||||
cli.Int64Flag{Name: "split-conns, x", Usage: "split download using multiple connections", EnvVar: "GOPHAST_SPLIT_CONNECTIONS", Value: 3},
|
||||
cli.Int64Flag{Name: "downloads, j", Usage: "number of files to download concurrently", EnvVar: "GOPHAST_DOWNLOADS", Value: 5},
|
||||
cli.BoolFlag{Name: "no-progress, n", Usage: "disable progress bars", EnvVar: "GOPHAST_VERBOSE"},
|
||||
cli.BoolFlag{Name: "quiet, q", Usage: "print error messages only", EnvVar: "GOPHAST_QUIET"},
|
||||
cli.BoolFlag{Name: "verbose, v", Usage: "print verbose messages (disables progress bars)", EnvVar: "GOPHAST_VERBOSE"},
|
||||
}
|
||||
sort.Sort(cli.FlagsByName(app.Flags))
|
||||
|
||||
app.Action = GoPhast
|
||||
|
||||
err := app.Run(os.Args)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func GoPhast(c *cli.Context) error {
|
||||
if !c.Bool("no-progress") && (isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) {
|
||||
config.ProgressLevel = ProgressDynamic
|
||||
}
|
||||
|
||||
config.MaxLogLevel = LogStandard
|
||||
if c.Bool("quiet") && c.Bool("verbose") {
|
||||
panic("--quiet and --verbose options are mutually exclusive")
|
||||
} else if c.Bool("quiet") {
|
||||
config.MaxLogLevel = LogNone
|
||||
} else if c.Bool("verbose") {
|
||||
config.MaxLogLevel = LogVerbose
|
||||
config.ProgressLevel = ProgressStatic
|
||||
}
|
||||
|
||||
if config.ProgressLevel == ProgressDynamic {
|
||||
progress = mpb.New(mpb.WithWidth(40), mpb.WithRefreshRate(120*time.Millisecond), mpb.WithWaitGroup(wg))
|
||||
}
|
||||
|
||||
var (
|
||||
postData []byte
|
||||
downloadName string
|
||||
downloadSize int64 = -1
|
||||
downloadDir = c.String("dir")
|
||||
downloadFileName = c.String("out")
|
||||
downloadFilePath string
|
||||
downloadFile *os.File
|
||||
downloadSplitSize = c.Int64("split-size")
|
||||
downloadURL = c.Args().First()
|
||||
|
||||
err error
|
||||
)
|
||||
|
||||
if downloadSplitSize < 0 {
|
||||
return errors.New("invalid minimum byte range request size (--min-split-size)")
|
||||
} else if downloadSplitSize == 0 {
|
||||
downloadSplitSize = DefaultMinimumSplitSize
|
||||
}
|
||||
|
||||
if downloadURL == "" {
|
||||
return c.App.Command("help").Run(c)
|
||||
}
|
||||
|
||||
if downloadDir == "" {
|
||||
downloadDir = "./"
|
||||
}
|
||||
downloadDir, err = filepath.Abs(downloadDir)
|
||||
CheckError(err)
|
||||
|
||||
logMessage(LogStandard, "Fetching metadata...")
|
||||
|
||||
downloadName, downloadSize = metadata(downloadURL, postData)
|
||||
|
||||
if downloadFileName == "" {
|
||||
downloadFileName = downloadName
|
||||
}
|
||||
|
||||
if filepath.IsAbs(downloadFileName) {
|
||||
downloadFilePath = downloadFileName
|
||||
} else {
|
||||
downloadFilePath = filepath.Join(downloadDir, downloadFileName)
|
||||
}
|
||||
|
||||
downloadFileName = filepath.Base(downloadFileName)
|
||||
|
||||
downloadFile, err = os.OpenFile(downloadFilePath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
CheckError(err)
|
||||
|
||||
defer downloadFile.Close()
|
||||
|
||||
if downloadSize == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
logMessage(LogStandard, "Allocating disk space...")
|
||||
|
||||
err = downloadFile.Truncate(downloadSize)
|
||||
CheckError(err)
|
||||
|
||||
splitCount := int64(1)
|
||||
if downloadSize >= (downloadSplitSize * 2) {
|
||||
splitCount = int64(math.Floor(float64(downloadSize) / float64(downloadSplitSize)))
|
||||
}
|
||||
|
||||
maxConnections := c.Int64("split-conns")
|
||||
if splitCount > maxConnections {
|
||||
splitCount = maxConnections
|
||||
}
|
||||
|
||||
if config.ProgressLevel == ProgressStatic {
|
||||
logMessage(LogStandard, "Downloading...")
|
||||
} else {
|
||||
log.Println() // Divider
|
||||
}
|
||||
|
||||
progressTaskFormat = "Range %" + strconv.Itoa(len(fmt.Sprintf("%d", splitCount))) + "d"
|
||||
progressTaskWidth = int(math.Max(math.Max(float64(len(downloadFileName)), float64(len(fmt.Sprintf(progressTaskFormat, 1)))), 11)) + 1
|
||||
|
||||
downloadSplitSize = downloadSize / splitCount
|
||||
downloadStart := time.Now()
|
||||
|
||||
var bar *mpb.Bar
|
||||
rangeStart := int64(0)
|
||||
rangeEnd := int64(0)
|
||||
for i := int64(1); i <= splitCount; i++ {
|
||||
if i < splitCount {
|
||||
rangeEnd += downloadSplitSize
|
||||
} else {
|
||||
rangeEnd = downloadSize - 1
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
if config.ProgressLevel == ProgressDynamic {
|
||||
task := fmt.Sprintf(progressTaskFormat, i)
|
||||
bar = progress.AddBar(rangeEnd-rangeStart+1,
|
||||
mpb.BarClearOnComplete(),
|
||||
mpb.PrependDecorators(
|
||||
decor.Name(task, decor.WC{W: progressTaskWidth, C: decor.DidentRight}),
|
||||
decor.OnComplete(decor.Name("downloading", decor.WCSyncSpaceR), "downloaded"),
|
||||
decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_HHMMSS, 60, decor.WCSyncWidth), "")),
|
||||
mpb.AppendDecorators(
|
||||
decor.OnComplete(decor.Percentage(decor.WC{W: 5}), ""),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
go download(downloadURL, postData, rangeStart, rangeEnd, downloadFile, i, splitCount, bar)
|
||||
|
||||
rangeStart = rangeEnd + 1
|
||||
}
|
||||
|
||||
if config.ProgressLevel == ProgressDynamic {
|
||||
progress.AddBar(0, mpb.BarStyle(" "), mpb.BarNoBrackets()).SetTotal(0, true) // Divider
|
||||
|
||||
b := progress.AddBar(downloadSize,
|
||||
mpb.BarRemoveOnComplete(),
|
||||
mpb.PrependDecorators(
|
||||
decor.Name(downloadFileName, decor.WC{W: progressTaskWidth, C: decor.DidentRight}),
|
||||
decor.OnComplete(decor.Name("downloading", decor.WCSyncSpaceR), "downloaded"),
|
||||
decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_HHMMSS, 60, decor.WCSyncWidth), "")),
|
||||
mpb.AppendDecorators(
|
||||
decor.OnComplete(decor.Percentage(decor.WC{W: 5}), ""),
|
||||
),
|
||||
)
|
||||
go handleProgress(b)
|
||||
progress.Wait()
|
||||
} else {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
logMessagef(LogStandard, "%s downloaded in %s (%s @ %s/s)", downloadFileName, time.Since(downloadStart).Truncate(time.Second), formatBytes(downloadSize), formatBytes(int64(float64(float64(downloadSize))/time.Since(downloadStart).Seconds())))
|
||||
|
||||
return nil
|
||||
}
|
||||
26
pkg/gophast/config.go
Normal file
26
pkg/gophast/config.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package gophast
|
||||
|
||||
import "git.sr.ht/~tslocum/gophast/pkg/progress"
|
||||
|
||||
type Config struct {
|
||||
MaxLogLevel LogLevel
|
||||
ProgressLevel progress.DisplayLevel
|
||||
|
||||
NoResume bool
|
||||
DownloadDir string
|
||||
DownloadName string
|
||||
|
||||
MinSplitSize int64
|
||||
MaxConnections int64
|
||||
}
|
||||
|
||||
var C = &Config{
|
||||
MaxLogLevel: LogNone,
|
||||
ProgressLevel: progress.None,
|
||||
|
||||
NoResume: false,
|
||||
DownloadDir: "./",
|
||||
DownloadName: "",
|
||||
|
||||
MinSplitSize: DefaultMinSplitSize,
|
||||
MaxConnections: DefaultMaxConnections}
|
||||
235
pkg/gophast/gophast.go
Normal file
235
pkg/gophast/gophast.go
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
package gophast
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.sr.ht/~tslocum/gophast/pkg/progress"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultMinSplitSize = 10 * 1024 * 1024 // 10 megabytes
|
||||
DefaultMaxConnections = 3
|
||||
)
|
||||
|
||||
var (
|
||||
wg = new(sync.WaitGroup)
|
||||
)
|
||||
|
||||
func request(method string, url string, postData []byte, rangeStart int64, rangeEnd int64) (*http.Response, error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest(method, url, bytes.NewReader(postData))
|
||||
CheckError(err)
|
||||
|
||||
req.Header.Add("User-Agent", "gophast https://git.sr.ht/~tslocum/gophast")
|
||||
if rangeEnd > 0 {
|
||||
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd))
|
||||
}
|
||||
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
func DownloadRange(url string, postData []byte, rangeStart int64, rangeEnd int64, file *os.File, id int64, maxid int64, localProgress chan<- *progress.BytesWrote) {
|
||||
var (
|
||||
wrote int64
|
||||
)
|
||||
defer wg.Done()
|
||||
start := time.Now()
|
||||
|
||||
rangeSize := rangeEnd - rangeStart + 1
|
||||
|
||||
resp, err := request("GET", url, postData, rangeStart, rangeEnd)
|
||||
CheckError(err)
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent {
|
||||
panic(fmt.Sprintf("invalid response code: %d (expected: 206 Partial Content)", resp.StatusCode))
|
||||
}
|
||||
|
||||
ow := NewOffsetWriter(file, rangeStart, localProgress)
|
||||
|
||||
wrote, err = io.Copy(ow, resp.Body)
|
||||
CheckError(err)
|
||||
if wrote != rangeSize {
|
||||
_ = resp.Body.Close()
|
||||
panic(fmt.Sprintf("received %d bytes of expected %d", wrote, rangeSize))
|
||||
}
|
||||
|
||||
err = resp.Body.Close()
|
||||
CheckError(err)
|
||||
|
||||
LogMessagef(LogVerbose, "Finished range %d/%d (%s in %s at %s/s)", id, maxid, formatBytes(rangeSize), formatDuration(time.Since(start)), formatBytes(int64(float64(float64(rangeSize))/time.Since(start).Seconds())))
|
||||
}
|
||||
|
||||
func Metadata(url string, postData []byte) (string, int64) {
|
||||
name := path.Base(url)
|
||||
|
||||
resp, err := request("HEAD", url, postData, 0, 0)
|
||||
CheckError(err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 404 {
|
||||
log.Fatal("404: not found")
|
||||
} else if resp.StatusCode != 200 {
|
||||
log.Fatalf("server responded with status %d (expected 200 OK)", resp.StatusCode)
|
||||
}
|
||||
|
||||
if resp.ContentLength < 0 {
|
||||
panic("server did not send Content-Length")
|
||||
}
|
||||
|
||||
if !strings.Contains(resp.Header.Get("Accept-Ranges"), "bytes") {
|
||||
panic("server does not support Range")
|
||||
}
|
||||
|
||||
// TODO: Handle redirect, update name
|
||||
|
||||
return name, resp.ContentLength
|
||||
}
|
||||
|
||||
func Download(url string, postData []byte, filePath string, size int64) bool {
|
||||
var localProgress chan<- *progress.BytesWrote
|
||||
if C.ProgressLevel == progress.Dynamic {
|
||||
progress.Initialize(40, wg)
|
||||
}
|
||||
|
||||
fileName := filepath.Base(filePath)
|
||||
|
||||
if !C.NoResume {
|
||||
if ex, err := os.Stat(filePath); err == nil {
|
||||
if ex.Size() == size {
|
||||
LogMessagef(LogStandard, "%s already downloaded", fileName)
|
||||
return true
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
downloadFile, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
CheckError(err)
|
||||
|
||||
defer downloadFile.Close()
|
||||
|
||||
if size == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
LogMessage(LogStandard, "Allocating disk space...")
|
||||
|
||||
err = downloadFile.Truncate(size)
|
||||
CheckError(err)
|
||||
|
||||
splitCount := int64(1)
|
||||
if C.MinSplitSize == 0 {
|
||||
if C.MaxConnections > 0 {
|
||||
splitCount = C.MaxConnections
|
||||
}
|
||||
} else if size >= (C.MinSplitSize * 2) {
|
||||
splitCount = int64(math.Floor(float64(size) / float64(C.MinSplitSize)))
|
||||
if splitCount > C.MaxConnections {
|
||||
splitCount = C.MaxConnections
|
||||
}
|
||||
}
|
||||
if splitCount > size {
|
||||
splitCount = size
|
||||
}
|
||||
|
||||
if C.ProgressLevel == progress.Static {
|
||||
LogMessage(LogStandard, "Starting download...")
|
||||
} else {
|
||||
log.Println() // Divider
|
||||
}
|
||||
|
||||
progress.TaskFormat = "Range %" + strconv.Itoa(len(fmt.Sprintf("%d", splitCount))) + "d"
|
||||
progress.TaskWidth = int(math.Max(math.Max(float64(len(fileName)), float64(len(fmt.Sprintf(progress.TaskFormat, 1)))), 11)) + 1
|
||||
|
||||
downloadSplitSize := size / splitCount
|
||||
downloadStart := time.Now()
|
||||
|
||||
rangeStart := int64(0)
|
||||
rangeEnd := int64(0)
|
||||
for i := int64(1); i <= splitCount; i++ {
|
||||
if i < splitCount {
|
||||
rangeEnd += downloadSplitSize
|
||||
} else {
|
||||
rangeEnd = size - 1
|
||||
}
|
||||
|
||||
LogMessagef(LogVerbose, "Starting range %d/%d (%d-%d)", i, splitCount, rangeStart, rangeEnd)
|
||||
wg.Add(1)
|
||||
|
||||
if C.ProgressLevel == progress.Dynamic {
|
||||
localProgress = progress.AddLocalBar(i, rangeEnd-rangeStart+1)
|
||||
}
|
||||
|
||||
go DownloadRange(url, postData, rangeStart, rangeEnd, downloadFile, i, splitCount, localProgress)
|
||||
|
||||
rangeStart = rangeEnd + 1
|
||||
}
|
||||
|
||||
if C.ProgressLevel == progress.Dynamic {
|
||||
progress.AddGlobalBar(fileName, size)
|
||||
|
||||
progress.Wait()
|
||||
} else {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
LogMessagef(LogStandard, "%s downloaded (%s in %s at %s/s)", fileName, formatBytes(size), formatDuration(time.Since(downloadStart)), formatBytes(int64(float64(float64(size))/time.Since(downloadStart).Seconds())))
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func DownloadPath(defaultFileName string) string {
|
||||
if C.DownloadName == "" {
|
||||
C.DownloadName = defaultFileName
|
||||
}
|
||||
if filepath.IsAbs(C.DownloadName) {
|
||||
return C.DownloadName
|
||||
}
|
||||
|
||||
downloadDir, err := filepath.Abs(C.DownloadDir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(downloadDir, C.DownloadName)
|
||||
}
|
||||
|
||||
func formatDuration(duration time.Duration) string {
|
||||
return fmt.Sprintf("%02d:%02d:%02d", int64((duration/time.Hour)%60), int64((duration/time.Minute)%60), int64((duration/time.Second)%60))
|
||||
}
|
||||
|
||||
func formatBytes(bytes int64) string {
|
||||
if bytes == 0 {
|
||||
return "0 B"
|
||||
}
|
||||
|
||||
k := float64(1024)
|
||||
sizes := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
|
||||
|
||||
i := math.Floor(math.Log(float64(bytes)) / math.Log(k))
|
||||
|
||||
format := "%.1f"
|
||||
if i == 0 {
|
||||
format = "%f"
|
||||
}
|
||||
|
||||
return fmt.Sprintf(format+" %s", float64(bytes)/math.Pow(k, i), sizes[int(i)])
|
||||
}
|
||||
|
||||
func CheckError(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
26
pkg/gophast/log.go
Normal file
26
pkg/gophast/log.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package gophast
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
type LogLevel int
|
||||
|
||||
const (
|
||||
LogNone LogLevel = iota
|
||||
LogStandard
|
||||
LogVerbose
|
||||
)
|
||||
|
||||
func LogMessage(level LogLevel, message string) {
|
||||
if level > C.MaxLogLevel {
|
||||
return
|
||||
}
|
||||
|
||||
log.Println(message)
|
||||
}
|
||||
|
||||
func LogMessagef(level LogLevel, format string, a ...interface{}) {
|
||||
LogMessage(level, fmt.Sprintf(format, a...))
|
||||
}
|
||||
39
pkg/gophast/offsetwriter.go
Normal file
39
pkg/gophast/offsetwriter.go
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
package gophast
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"git.sr.ht/~tslocum/gophast/pkg/progress"
|
||||
)
|
||||
|
||||
type OffsetWriter struct {
|
||||
io.WriterAt
|
||||
offset int64
|
||||
|
||||
updateProgress chan<- *progress.BytesWrote
|
||||
lastUpdate time.Time
|
||||
}
|
||||
|
||||
func NewOffsetWriter(w io.WriterAt, offset int64, updateProgress chan<- *progress.BytesWrote) *OffsetWriter {
|
||||
return &OffsetWriter{WriterAt: w, offset: offset, updateProgress: updateProgress, lastUpdate: time.Now()}
|
||||
}
|
||||
|
||||
func (dst *OffsetWriter) Write(b []byte) (n int, err error) {
|
||||
n, err = dst.WriteAt(b, dst.offset)
|
||||
dst.offset += int64(n)
|
||||
|
||||
if dst.updateProgress == nil {
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case dst.updateProgress <- &progress.BytesWrote{Wrote: n, Duration: time.Since(dst.lastUpdate)}:
|
||||
default:
|
||||
log.Fatal("starved writer")
|
||||
}
|
||||
dst.lastUpdate = time.Now()
|
||||
|
||||
return
|
||||
}
|
||||
108
pkg/progress/progress.go
Normal file
108
pkg/progress/progress.go
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
package progress
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/vbauerster/mpb/v4/decor"
|
||||
|
||||
"github.com/vbauerster/mpb/v4"
|
||||
)
|
||||
|
||||
const BufferSize = 10
|
||||
|
||||
type DisplayLevel int
|
||||
|
||||
const (
|
||||
None DisplayLevel = iota
|
||||
Static
|
||||
Dynamic
|
||||
)
|
||||
|
||||
var (
|
||||
p *mpb.Progress
|
||||
globalProgress chan *BytesWrote
|
||||
|
||||
TaskWidth int
|
||||
TaskFormat string
|
||||
)
|
||||
|
||||
type BytesWrote struct {
|
||||
Wrote int
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
func Initialize(width int, wg *sync.WaitGroup) <-chan *BytesWrote {
|
||||
if p != nil {
|
||||
return globalProgress
|
||||
}
|
||||
|
||||
p = mpb.New(mpb.WithWidth(width), mpb.WithRefreshRate(120*time.Millisecond), mpb.WithWaitGroup(wg))
|
||||
globalProgress = make(chan *BytesWrote, BufferSize)
|
||||
|
||||
return globalProgress
|
||||
}
|
||||
|
||||
func AddLocalBar(index int64, total int64) chan<- *BytesWrote {
|
||||
task := fmt.Sprintf(TaskFormat, index)
|
||||
bar := p.AddBar(total,
|
||||
mpb.BarClearOnComplete(),
|
||||
mpb.PrependDecorators(
|
||||
decor.Name(task, decor.WC{W: TaskWidth, C: decor.DidentRight}),
|
||||
decor.OnComplete(decor.Name("downloading", decor.WCSyncSpaceR), "downloaded"),
|
||||
decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_HHMMSS, 60, decor.WCSyncWidth), "")),
|
||||
mpb.AppendDecorators(
|
||||
decor.OnComplete(decor.Percentage(decor.WC{W: 5}), ""),
|
||||
),
|
||||
)
|
||||
|
||||
localProgress := make(chan *BytesWrote, BufferSize)
|
||||
go HandleLocalBar(bar, localProgress)
|
||||
return localProgress
|
||||
}
|
||||
|
||||
func AddGlobalBar(fileName string, total int64) {
|
||||
// Divider
|
||||
p.AddBar(0, mpb.BarStyle(" "), mpb.BarNoBrackets()).SetTotal(0, true)
|
||||
|
||||
globalBar := p.AddBar(total,
|
||||
mpb.BarRemoveOnComplete(),
|
||||
mpb.PrependDecorators(
|
||||
decor.Name(fileName, decor.WC{W: TaskWidth, C: decor.DidentRight}),
|
||||
decor.OnComplete(decor.Name("downloading", decor.WCSyncSpaceR), "downloaded"),
|
||||
decor.OnComplete(decor.EwmaETA(decor.ET_STYLE_HHMMSS, 60, decor.WCSyncWidth), "")),
|
||||
mpb.AppendDecorators(
|
||||
decor.OnComplete(decor.Percentage(decor.WC{W: 5}), ""),
|
||||
),
|
||||
)
|
||||
|
||||
go HandleGlobalBar(globalBar)
|
||||
}
|
||||
|
||||
func HandleLocalBar(bar *mpb.Bar, localProgress <-chan *BytesWrote) {
|
||||
var bw *BytesWrote
|
||||
for {
|
||||
bw = <-localProgress
|
||||
|
||||
bar.IncrBy(bw.Wrote, bw.Duration)
|
||||
|
||||
globalProgress <- bw
|
||||
}
|
||||
}
|
||||
func HandleGlobalBar(bar *mpb.Bar) {
|
||||
var bw *BytesWrote
|
||||
for {
|
||||
bw = <-globalProgress
|
||||
|
||||
bar.IncrBy(bw.Wrote, bw.Duration)
|
||||
}
|
||||
}
|
||||
|
||||
func Wait() {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
|
||||
p.Wait()
|
||||
}
|
||||
Loading…
Reference in a new issue