[Golang] Read Lines From URL


First show how to readlines from URL, and then extend to read lines from file and string.

readlines from URL

url.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
package lines

import (
	"bufio"
	"io"
	"net/http"
)

func UrlToLines(url string) ([]string, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return LinesFromReader(resp.Body)
}

func LinesFromReader(r io.Reader) ([]string, error) {
	var lines []string
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

Note that LinesFromReader func accepts argument of io.Reader, which is an interface. Later this function will be re-used to read lines from file/string.

Usage of UrlToLines:

url_test.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
package lines

import "testing"

func TestUrlToLines(t *testing.T) {
	lines, err := UrlToLines("https://raw.githubusercontent.com/siongui/userpages/master/README.rst")
	if err != nil {
		t.Error(err)
	}

	for _, line := range lines {
		println(line)
	}
}

readlines from file

Use the following code with the LinesFromReader func in the previous section, we can read a file line by line:

file.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
package lines

import "os"

func FileToLines(filePath string) ([]string, error) {
	f, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return LinesFromReader(f)
}

Usage of FileToLines:

file_test.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
package lines

import "testing"

func TestFileToLines(t *testing.T) {
	lines, err := FileToLines("file.go")
	if err != nil {
		t.Error(err)
	}

	for _, line := range lines {
		println(line)
	}
}

readlines from string

Use the following code with the LinesFromReader func in the previous section, we can read a string line by line:

string.go | repository | view raw
1
2
3
4
5
6
7
8
package lines

import "strings"

func StringToLines(s string) ([]string, error) {
	r := strings.NewReader(s)
	return LinesFromReader(r)
}

Usage of StringToLines:

string_test.go | repository | view raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
package lines

import "testing"

func TestStringToLines(t *testing.T) {
	str := "hello\nworld\n"
	lines, err := StringToLines(str)
	if err != nil {
		t.Error(err)
	}

	for _, line := range lines {
		println(line)
	}
}

Tested on:

  • Ubuntu Linux 16.10
  • Go 1.7.5

References:

[1][Golang] Read Lines From File or String
[2]read lines from file or url · siongui/go-rst@f0cc55b · GitHub
[3]Processing of huge text files : golang
[4]Tips to working with large CSV files? : golang