[Golang] Download HTML From URL


Download and save HTML file from given URL via Golang. Do nothing if the HTML file already locally exists.

download.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
package main

import (
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
)

func download(url, filename string) (err error) {
	fmt.Println("Downloading ", url, " to ", filename)

	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	f, err := os.Create(filename)
	if err != nil {
		return
	}
	defer f.Close()

	_, err = io.Copy(f, resp.Body)
	return
}

func main() {
	pUrl := flag.String("url", "", "URL to be processed")
	flag.Parse()
	url := *pUrl
	if url == "" {
		fmt.Fprintf(os.Stderr, "Error: empty URL!\n")
		return
	}

	filename := path.Base(url)
	fmt.Println("Checking if " + filename + " exists ...")
	if _, err := os.Stat(filename); os.IsNotExist(err) {
		err := download(url, filename)
		if err != nil {
			panic(err)
		}
		fmt.Println(filename + " saved!")
	} else {
		fmt.Println(filename + " already exists!")
	}
}
Makefile | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
export GOROOT=$(realpath ../../../../../go)
export GOPATH=$(realpath .)
export PATH := $(GOROOT)/bin:$(GOPATH)/bin:$(PATH)

URL=https://siongui.github.io/index.html

default:
	@echo "\033[92mProcessing ${URL}...\033[0m"
	@go run download.go -url=${URL}

fmt:
	@echo "\033[92mGo fmt source code...\033[0m"
	@go fmt *.go

Tested on: Ubuntu Linux 18.04, Go 1.10.1.


References:

[1]
[2]http - The Go Programming Language
[3]
[4][Golang] Download File From URL
[5]save image from url at DuckDuckGo
[6]download file from url - Google Search