diff --git a/lib/docker/Dockerfile b/lib/docker/Dockerfile index b4c27ac2..0b70cc09 100644 --- a/lib/docker/Dockerfile +++ b/lib/docker/Dockerfile @@ -1,20 +1,17 @@ -# Examples: +# To build the image: # -# docker build -t ghcr.io/go-rod/rod -f lib/docker/Dockerfile . +# docker build -t ghcr.io/go-rod/rod -f lib/docker/Dockerfile . # -# // use mirrors -# docker build --build-arg goproxy=https://goproxy.io,direct --build-arg apt_sources=https://mirrors.tuna.tsinghua.edu.cn \ -# -t ghcr.io/go-rod/rod -f lib/docker/Dockerfile . -# build rod-launcher tool +# build rod-manager FROM golang AS go -ARG goproxy="" +ARG goproxy="https://goproxy.io,direct" COPY . /rod WORKDIR /rod RUN go env -w GO111MODULE=on && go env -w GOPROXY=$goproxy -RUN go build ./lib/launcher/rod-launcher +RUN go build ./lib/launcher/rod-manager RUN go run ./lib/utils/get-browser FROM ubuntu:bionic @@ -49,6 +46,6 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_sources|g" /etc/apt/sources.list && ENTRYPOINT ["dumb-init", "--"] COPY --from=go /root/.cache/rod /root/.cache/rod -COPY --from=go /rod/rod-launcher /usr/bin/ +COPY --from=go /rod/rod-manager /usr/bin/ -CMD rod-launcher +CMD rod-manager diff --git a/lib/examples/launch-managed/main.go b/lib/examples/launch-managed/main.go new file mode 100644 index 00000000..41b5ce34 --- /dev/null +++ b/lib/examples/launch-managed/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/utils" +) + +func main() { + // This example is to launch a browser remotely, not connect to a running browser remotely, + // to connect to a running browser check the "../connect-browser" example. + // Rod provides a docker image for beginers, run the below to start a launcher.Manager: + // + // docker run -p 7317:7317 ghcr.io/go-rod/rod + // + // For more information, check the doc of launcher.Manager + l := launcher.MustNewManaged("") + + // You can also set any flag remotely before you launch the remote browser. + // Available flags: https://peter.sh/experiments/chromium-command-line-switches + l.Set("disable-gpu").Delete("disable-gpu") + + // Launch with headful mode + l.Headless(false).XVFB("--server-num=5", "--server-args=-screen 0 1600x900x16") + + browser := rod.New().Client(l.Client()).MustConnect() + + // You may want to start a server to watch the screenshots of the remote browser. + launcher.Open(browser.ServeMonitor("")) + + fmt.Println( + browser.MustPage("https://example.com/").MustEval("() => document.title"), + ) + + utils.Pause() +} diff --git a/lib/examples/remote-launch/main.go b/lib/examples/remote-launch/main.go deleted file mode 100644 index 4fdfcdee..00000000 --- a/lib/examples/remote-launch/main.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/go-rod/rod" - "github.com/go-rod/rod/lib/launcher" - "github.com/go-rod/rod/lib/utils" -) - -func main() { - // To launch remote browsers, you must use a remote launcher service, - // Don't launch the browser manually like "chrome --headless --remote-debugging-port=9222". - // To connect to a running browser check the "../connect-browser" example. - // Rod provides a docker image for beginers, run the below first: - // - // docker run -p 7317:7317 ghcr.io/go-rod/rod - // - // For more information, check the doc of launcher.RemoteLauncher - l := launcher.MustNewRemote("") - - // Manipulate flags like the example in examples_test.go - l.Set("any-flag").Delete("any-flag") - - // Launch with headful mode - l.Headless(false).XVFB("--server-num=5", "--server-args=-screen 0 1600x900x16") - - browser := rod.New().Client(l.Client()).MustConnect() - - // You may want to start a server to watch the screenshots inside the docker - launcher.Open(browser.ServeMonitor("")) - - fmt.Println( - browser.MustPage("https://example.com/").MustEval("() => document.title"), - ) - - utils.Pause() -} diff --git a/lib/launcher/launcher.go b/lib/launcher/launcher.go index 544cfd40..b3121e3a 100644 --- a/lib/launcher/launcher.go +++ b/lib/launcher/launcher.go @@ -21,22 +21,26 @@ const ( flagWorkingDir = "rod-working-dir" flagEnv = "rod-env" flagXVFB = "rod-xvfb" + flagLeakless = "rod-leakless" + flagBin = "rod-bin" ) // Launcher is a helper to launch browser binary smartly type Launcher struct { - logger io.Writer + Flags map[string][]string `json:"flags"` + ctx context.Context ctxCancel func() - browser *Browser - bin string - url string - parser *URLParser - Flags map[string][]string `json:"flags"` - pid int - exit chan struct{} - remote bool // remote mode or not - leakless bool + + logger io.Writer + + browser *Browser + parser *URLParser + pid int + exit chan struct{} + + managed bool + serviceURL string } // New returns the default arguments to start browser. @@ -50,6 +54,9 @@ func New() *Launcher { } defaultFlags := map[string][]string{ + flagBin: {defaults.Bin}, + flagLeakless: nil, + "user-data-dir": {dir}, // use random port by default @@ -109,9 +116,7 @@ func New() *Launcher { Flags: defaultFlags, exit: make(chan struct{}), browser: NewBrowser(), - bin: defaults.Bin, parser: NewURLParser(), - leakless: true, logger: ioutil.Discard, } } @@ -129,10 +134,10 @@ func NewUserMode() *Launcher { Flags: map[string][]string{ "remote-debugging-port": {"37712"}, "no-startup-window": nil, + flagBin: {bin}, }, browser: NewBrowser(), exit: make(chan struct{}), - bin: bin, parser: NewURLParser(), logger: ioutil.Discard, } @@ -159,6 +164,12 @@ func (l *Launcher) Get(name string) (string, bool) { return "", false } +// Has flag or not +func (l *Launcher) Has(name string) bool { + _, has := l.GetFlags(name) + return has +} + // GetFlags from settings func (l *Launcher) GetFlags(name string) ([]string, bool) { flag, has := l.Flags[l.normalizeFlag(name)] @@ -189,8 +200,7 @@ func (l *Launcher) Delete(name string) *Launcher { // Bin set browser executable file path. If it's empty, launcher will automatically search or download the bin. func (l *Launcher) Bin(path string) *Launcher { - l.bin = path - return l + return l.Set(flagBin, path) } // Headless switch. Whether to run browser in headless mode. A mode without visible UI. @@ -220,8 +230,10 @@ func (l *Launcher) XVFB(args ...string) *Launcher { // Leakless switch. If enabled, the browser will be force killed after the Go process exits. // The doc of leakless: https://github.com/ysmood/leakless. func (l *Launcher) Leakless(enable bool) *Launcher { - l.leakless = enable - return l + if enable { + return l.Set(flagLeakless) + } + return l.Delete(flagLeakless) } // Devtools switch to auto open devtools for each tab @@ -257,12 +269,8 @@ func (l *Launcher) ProfileDir(dir string) *Launcher { } // RemoteDebuggingPort to launch the browser. Zero for a random port. Zero is the default value. -// If it's not zero, the launcher will try to connect to it before starting a new browser process. -// For example, to reuse the same browser process for between 2 runs of a Go program, you can -// do something like: -// launcher.New().RemoteDebuggingPort(9222).MustLaunch() -// -// Related doc: https://chromedevtools.github.io/devtools-protocol/ +// If it's not zero and the Launcher.Leakless is disabled, the launcher will try to reconnect to it first, +// if the reconnection fails it will launch a new browser. func (l *Launcher) RemoteDebuggingPort(port int) *Launcher { return l.Set("remote-debugging-port", fmt.Sprintf("%d", port)) } @@ -345,7 +353,7 @@ func (l *Launcher) Launch() (string, error) { var ll *leakless.Launcher var cmd *exec.Cmd - if l.leakless && leakless.Support() { + if l.Has(flagLeakless) && leakless.Support() { ll = leakless.New() cmd = ll.Command(bin, l.FormatArgs()...) } else { @@ -400,11 +408,12 @@ func (l *Launcher) setupCmd(cmd *exec.Cmd) { } func (l *Launcher) getBin() (string, error) { - if l.bin == "" { + bin, _ := l.Get(flagBin) + if bin == "" { l.browser.Context = l.ctx return l.browser.Get() } - return l.bin, nil + return bin, nil } func (l *Launcher) getURL() (u string, err error) { diff --git a/lib/launcher/launcher_test.go b/lib/launcher/launcher_test.go index f50a17b8..5eec2a43 100644 --- a/lib/launcher/launcher_test.go +++ b/lib/launcher/launcher_test.go @@ -103,13 +103,13 @@ func (t T) Launch() { } { - _, err := launcher.NewRemote("") + _, err := launcher.NewManaged("") t.Err(err) - _, err = launcher.NewRemote("1://") + _, err = launcher.NewManaged("1://") t.Err(err) - _, err = launcher.NewRemote("ws://not-exists") + _, err = launcher.NewManaged("ws://not-exists") t.Err(err) } } diff --git a/lib/launcher/load_test.go b/lib/launcher/load_test.go index 2b5f3fea..c6410b7b 100644 --- a/lib/launcher/load_test.go +++ b/lib/launcher/load_test.go @@ -12,7 +12,7 @@ import ( "github.com/ysmood/got" ) -func BenchmarkRemoteLauncher(b *testing.B) { +func BenchmarkManager(b *testing.B) { const concurrent = 30 // how many browsers will run at the same time const num = 300 // how many browsers we will launch @@ -47,7 +47,7 @@ func BenchmarkRemoteLauncher(b *testing.B) { }() }() - l := launcher.MustNewRemote("") + l := launcher.MustNewManaged("") client := l.Client() browser := rod.New().Context(ctx).Client(client).MustConnect() page := browser.MustPage() diff --git a/lib/launcher/manager.go b/lib/launcher/manager.go new file mode 100644 index 00000000..29944d18 --- /dev/null +++ b/lib/launcher/manager.go @@ -0,0 +1,162 @@ +package launcher + +import ( + "encoding/json" + "net/http" + "net/http/httputil" + "net/url" + + "github.com/go-rod/rod/lib/cdp" + "github.com/go-rod/rod/lib/utils" +) + +// HeaderName for remote launch +const HeaderName = "Rod-Launcher" + +const flagKeepUserDataDir = "rod-keep-user-data-dir" + +// MustNewManaged is similar to MustNewManaged +func MustNewManaged(serviceURL string) *Launcher { + l, err := NewManaged(serviceURL) + utils.E(err) + return l +} + +// NewManaged creates a default Launcher instance from launcher.Manager. +// The serviceURL must point to a launcher.Manager. It will send a http request to the serviceURL +// to get the default settings of the Launcher instance. For example if the launcher.Manager running on a +// Linux machine will return different default settings from the one on Mac. +// If Launcher.Leakless is enabled, the remote browser will be killed after the websocket is closed. +func NewManaged(serviceURL string) (*Launcher, error) { + if serviceURL == "" { + serviceURL = "ws://127.0.0.1:7317" + } + + u, err := url.Parse(serviceURL) + if err != nil { + return nil, err + } + + l := New() + l.managed = true + l.serviceURL = toWS(*u).String() + l.Flags = nil + + res, err := http.Get(toHTTP(*u).String()) + if err != nil { + return nil, err + } + + return l, json.NewDecoder(res.Body).Decode(l) +} + +// KeepUserDataDir after remote browser is closed. By default user-data-dir will be removed. +func (l *Launcher) KeepUserDataDir() *Launcher { + l.mustManaged() + l.Set(flagKeepUserDataDir) + return l +} + +// JSON serialization +func (l *Launcher) JSON() []byte { + return utils.MustToJSONBytes(l) +} + +// Client for launching browser remotely, such as browser from a docker container. +func (l *Launcher) Client() *cdp.Client { + l.mustManaged() + header := http.Header{} + header.Add(HeaderName, utils.MustToJSON(l)) + return cdp.New(l.serviceURL).Header(header) +} + +func (l *Launcher) mustManaged() { + if !l.managed { + panic("Must be used with launcher.NewManaged") + } +} + +var _ http.Handler = &Manager{} + +// Manager is used to launch browsers via http server on another machine. +// The reason why we have Manager is after we launcher a browser, we can't dynamicall change its +// CLI arguments, such as "--headless". The Manager allows us to decide what CLI arguments to +// pass to the browser when launch it remotely. +// The work flow looks like: +// +// | Machine X | Machine Y | +// | NewManaged("a.com") -|-> http.ListenAndServe("a.com", launcher.NewManager()) --> launch browser | +// +// 1. X send a http request to Y, Y respond default Launcher settings based the OS of Y. +// 2. X start a websocket connect to Y with the Launcher settings +// 3. Y launches a browser with the Launcher settings X +// 4. Y transparently proxy the websocket connect between X and the launched browser +// +type Manager struct { + Logger utils.Logger + Defaults func() *Launcher +} + +// NewManager instance +func NewManager() *Manager { + return &Manager{ + Logger: utils.LoggerQuiet, + Defaults: New, + } +} + +func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Upgrade") == "websocket" { + m.launch(w, r) + return + } + + m.defaults(w, r) +} + +func (m *Manager) defaults(w http.ResponseWriter, _ *http.Request) { + l := New() + utils.E(w.Write(l.JSON())) +} + +func (m *Manager) launch(w http.ResponseWriter, r *http.Request) { + l := m.Defaults() + + options := r.Header.Get(HeaderName) + if options != "" { + l.Flags = nil + utils.E(json.Unmarshal([]byte(options), l)) + } + + kill := l.Has(flagLeakless) + + // Always enable leakless so that if the Manager process crashes + // all the managed browsers will be killed. + u := l.Leakless(true).MustLaunch() + defer m.cleanup(l, kill) + + parsedURL, err := url.Parse(u) + utils.E(err) + + m.Logger.Println("Launch", u, options) + defer m.Logger.Println("Close", u) + + parsedWS, err := url.Parse(u) + utils.E(err) + parsedURL.Path = parsedWS.Path + + httputil.NewSingleHostReverseProxy(toHTTP(*parsedURL)).ServeHTTP(w, r) +} + +func (m *Manager) cleanup(l *Launcher, kill bool) { + if kill { + l.Kill() + m.Logger.Println("Killed PID:", l.PID()) + } + + if !l.Has(flagKeepUserDataDir) { + l.Cleanup() + dir, _ := l.Get("user-data-dir") + m.Logger.Println("Removed", dir) + } +} diff --git a/lib/launcher/private_test.go b/lib/launcher/private_test.go index e02b44de..7ef976d0 100644 --- a/lib/launcher/private_test.go +++ b/lib/launcher/private_test.go @@ -102,10 +102,10 @@ func (t T) RemoteLaunch() { defer cancel() s := got.New(t).Serve() - rl := NewRemoteLauncher() + rl := NewManager() s.Mux.Handle("/", rl) - l := MustNewRemote(s.URL()).KeepUserDataDir().Delete(flagKeepUserDataDir) + l := MustNewManaged(s.URL()).KeepUserDataDir().Delete(flagKeepUserDataDir) client := l.Client() b := client.MustConnect(ctx) t.E(b.Call(ctx, "", "Browser.getVersion", nil)) diff --git a/lib/launcher/remote_launcher.go b/lib/launcher/remote_launcher.go deleted file mode 100644 index be5a079c..00000000 --- a/lib/launcher/remote_launcher.go +++ /dev/null @@ -1,146 +0,0 @@ -package launcher - -import ( - "encoding/json" - "net/http" - "net/http/httputil" - "net/url" - - "github.com/go-rod/rod/lib/cdp" - "github.com/go-rod/rod/lib/utils" -) - -// HeaderName for remote launch -const HeaderName = "Rod-Launcher" - -const flagKeepUserDataDir = "rod-keep-user-data-dir" - -// MustNewRemote is similar to NewRemote -func MustNewRemote(remoteURL string) *Launcher { - l, err := NewRemote(remoteURL) - utils.E(err) - return l -} - -// NewRemote creates a Launcher instance from remote defaults. -// The browser it connects to must be launched by RemoteLauncher. -// For more info check the doc of RemoteLauncher. -func NewRemote(remoteURL string) (*Launcher, error) { - if remoteURL == "" { - remoteURL = "ws://127.0.0.1:7317" - } - - u, err := url.Parse(remoteURL) - if err != nil { - return nil, err - } - - l := New() - l.remote = true - l.url = toWS(*u).String() - l.Flags = nil - - res, err := http.Get(toHTTP(*u).String()) - if err != nil { - return nil, err - } - - return l, json.NewDecoder(res.Body).Decode(l) -} - -// KeepUserDataDir after remote browser is closed. By default user-data-dir will be removed. -func (l *Launcher) KeepUserDataDir() *Launcher { - l.mustRemote() - l.Set(flagKeepUserDataDir) - return l -} - -// JSON serialization -func (l *Launcher) JSON() []byte { - return utils.MustToJSONBytes(l) -} - -// Client for launching browser remotely, such as browser from a docker container. -func (l *Launcher) Client() *cdp.Client { - l.mustRemote() - header := http.Header{} - header.Add(HeaderName, utils.MustToJSON(l)) - return cdp.New(l.url).Header(header) -} - -func (l *Launcher) mustRemote() { - if !l.remote { - panic("Must be used with launcher.NewRemote") - } -} - -var _ http.Handler = &RemoteLauncher{} - -// RemoteLauncher is used to launch browsers via http server on another machine. -// For example, the work flow looks like: -// -// | Machine A | Machine B | -// NewRemote("a.com") --> http.ListenAndServe("a.com", NewRemoteLauncher()) --> launch browser -// -// Any http request will return a default Launcher based on remote OS environment. -// Any websocket request will start a new browser and the request will be proxied to the browser. -// The websocket header "Rod-Launcher" holds the options to launch browser. -// If the websocket is closed, the browser will be killed. -type RemoteLauncher struct { - Logger utils.Logger -} - -// NewRemoteLauncher instance -func NewRemoteLauncher() *RemoteLauncher { - return &RemoteLauncher{ - Logger: utils.LoggerQuiet, - } -} - -func (p *RemoteLauncher) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Upgrade") == "websocket" { - p.launch(w, r) - return - } - - p.defaults(w, r) -} - -func (p *RemoteLauncher) defaults(w http.ResponseWriter, _ *http.Request) { - l := New() - utils.E(w.Write(l.JSON())) -} - -func (p *RemoteLauncher) launch(w http.ResponseWriter, r *http.Request) { - l := New() - - options := r.Header.Get(HeaderName) - if options != "" { - l.Flags = nil - utils.E(json.Unmarshal([]byte(options), l)) - } - - u := l.Leakless(false).MustLaunch() - defer func() { - l.Kill() - p.Logger.Println("Killed PID:", l.PID()) - - if _, has := l.Get(flagKeepUserDataDir); !has { - l.Cleanup() - dir, _ := l.Get("user-data-dir") - p.Logger.Println("Removed", dir) - } - }() - - parsedURL, err := url.Parse(u) - utils.E(err) - - p.Logger.Println("Launch", u, options) - defer p.Logger.Println("Close", u) - - parsedWS, err := url.Parse(u) - utils.E(err) - parsedURL.Path = parsedWS.Path - - httputil.NewSingleHostReverseProxy(toHTTP(*parsedURL)).ServeHTTP(w, r) -} diff --git a/lib/launcher/rod-launcher/main.go b/lib/launcher/rod-launcher/main.go deleted file mode 100644 index ff481db9..00000000 --- a/lib/launcher/rod-launcher/main.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "net" - "net/http" - "os" - - "github.com/go-rod/rod/lib/launcher" - "github.com/go-rod/rod/lib/utils" -) - -var addr = flag.String("address", ":7317", "the address to listen to") -var quiet = flag.Bool("quiet", false, "silent the log") - -// a cli tool to launch browser remotely -func main() { - flag.Parse() - - rl := launcher.NewRemoteLauncher() - if !*quiet { - rl.Logger = log.New(os.Stdout, "", 0) - } - - l, err := net.Listen("tcp", *addr) - if err != nil { - utils.E(err) - } - - fmt.Println("Remote control url is", "ws://"+l.Addr().String()) - - srv := &http.Server{Handler: rl} - utils.E(srv.Serve(l)) -} diff --git a/lib/launcher/rod-manager/main.go b/lib/launcher/rod-manager/main.go new file mode 100644 index 00000000..7da8cc03 --- /dev/null +++ b/lib/launcher/rod-manager/main.go @@ -0,0 +1,44 @@ +// A server to help launch browser remotely +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/http" + "os" + + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/utils" +) + +var addr = flag.String("address", ":7317", "the address to listen to") +var quiet = flag.Bool("quiet", false, "silence the log") +var bin = flag.String("bin", "", "default browser executable path") + +func main() { + flag.Parse() + + m := launcher.NewManager() + + if !*quiet { + m.Logger = log.New(os.Stdout, "", 0) + } + + m.Defaults = func() *launcher.Launcher { + return launcher.New().Bin(*bin) + } + + l, err := net.Listen("tcp", *addr) + if err != nil { + utils.E(err) + } + + if !*quiet { + fmt.Println("rod-manager listening on:", l.Addr().String()) + } + + srv := &http.Server{Handler: m} + utils.E(srv.Serve(l)) +} diff --git a/lib/utils/docker/main.go b/lib/utils/docker/main.go index 0999aa12..578a7bf0 100644 --- a/lib/utils/docker/main.go +++ b/lib/utils/docker/main.go @@ -59,7 +59,7 @@ func test() { wd, err := os.Getwd() utils.E(err) - utils.Exec("docker", "run", image, "rod-launcher", "-h") + utils.Exec("docker", "run", image, "rod-manager", "-h") utils.Exec("docker", "run", "-v", fmt.Sprintf("%s:/t", wd), "-w=/t", "dev", "go", "test") } diff --git a/lib/utils/simple-check/main.go b/lib/utils/simple-check/main.go index a3b32a2d..7b28745a 100644 --- a/lib/utils/simple-check/main.go +++ b/lib/utils/simple-check/main.go @@ -9,6 +9,6 @@ func main() { utils.ExecLine("go test -coverprofile=coverage.txt ./lib/launcher") utils.ExecLine("go run ./lib/utils/check-cov") - utils.ExecLine("go test -coverprofile=coverage.txt -parallel=4") + utils.ExecLine("go test -coverprofile=coverage.txt") utils.ExecLine("go run ./lib/utils/check-cov") }