I heard the news of headless Chrome from HN , and found that it is possible
to write a Go program to automatically web-scraping JavaScript rendered web
pages via Chrome Debugging Protocol . I have tried to use Python dryscrape
to programmatically login a website before , and now I will try to use Go
chromedp package to login Facebook and take screenshot programmatically.
Install Go chromedp package for programmatically drive the browser and
simulate human user actions:
go get -u github.com/knq/chromedp
Install Go package for read password from console :
go get -u golang.org/x/crypto/ssh/terminal
Login Facebook and Take Screenshot :
main.go |
repository |
view raw
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 package main
import (
"context"
"fmt"
"io/ioutil"
"log"
cdp "github.com/knq/chromedp"
cdptypes "github.com/knq/chromedp/cdp"
"golang.org/x/crypto/ssh/terminal"
"syscall"
)
func getCredentials () ( u , p string ) {
fmt . Print ( "Username: " )
_ , err := fmt . Scan ( & u )
if err != nil {
panic ( err )
}
fmt . Print ( "Password: " )
password , err := terminal . ReadPassword ( int ( syscall . Stdin ))
if err != nil {
panic ( err )
}
p = string ( password )
return
}
func facebookLogin ( username , password string ) cdp . Tasks {
selname := `//input[@id="email"]`
selpass := `//input[@id="pass"]`
var buf [] byte
return cdp . Tasks {
cdp . Navigate ( `https://www.facebook.com` ),
cdp . WaitVisible ( selpass ),
cdp . SendKeys ( selname , username ),
cdp . SendKeys ( selpass , password ),
cdp . Submit ( selpass ),
cdp . WaitVisible ( `//a[@title="Profile"]` ),
cdp . CaptureScreenshot ( & buf ),
cdp . ActionFunc ( func ( context . Context , cdptypes . Handler ) error {
return ioutil . WriteFile ( "myfb.png" , buf , 0644 )
}),
}
}
func main () {
var err error
// create context
ctxt , cancel := context . WithCancel ( context . Background ())
defer cancel ()
// create chrome instance
c , err := cdp . New ( ctxt , cdp . WithLog ( log . Printf ))
if err != nil {
log . Fatal ( err )
}
// run task list
err = c . Run ( ctxt , facebookLogin ( getCredentials ()))
if err != nil {
log . Fatal ( err )
}
// shutdown chrome
err = c . Shutdown ( ctxt )
if err != nil {
log . Fatal ( err )
}
// wait for chrome to finish
err = c . Wait ()
if err != nil {
log . Fatal ( err )
}
}
Tested on: Ubuntu Linux 17.04 , Go 1.8.1 .
References: