From 75a94e0553e92fe090eafe57abaad8cb42c5ec67 Mon Sep 17 00:00:00 2001 From: Moses Narrow <36607567+0pcom@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:16:54 -0500 Subject: [PATCH 01/15] do not serve redundantly on local port with dmsg web srv --- cmd/dmsgweb/commands/dmsgweb.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index fc79bb48..e821b8db 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -311,7 +311,6 @@ var ( wl string wlkeys []cipher.PubKey localPort uint - websrvPort uint err error ) @@ -322,7 +321,6 @@ var dmsgwebsrvconffile = os.Getenv(dmsgwebsrvenvname) func init() { RootCmd.AddCommand(srvCmd) srvCmd.Flags().UintVarP(&localPort, "lport", "l", scriptExecUint("${LOCALPORT:-8086}", dmsgwebsrvconffile), "local application http interface port") - srvCmd.Flags().UintVarP(&websrvPort, "port", "p", scriptExecUint("${WEBPORT:-8081}", dmsgwebsrvconffile), "port to serve") srvCmd.Flags().UintVarP(&dmsgPort, "dport", "d", scriptExecUint("${DMSGPORT:-80}", dmsgwebsrvconffile), "dmsg port to serve") srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") @@ -466,13 +464,6 @@ func server() { wg.Done() }() - wg.Add(1) - go func() { - fmt.Printf("listening on http://127.0.0.1:%d using gin router\n", websrvPort) - r1.Run(fmt.Sprintf(":%d", websrvPort)) //nolint - wg.Done() - }() - wg.Wait() } @@ -783,9 +774,6 @@ const srvenvfileLinux = ` #-- DMSG port to serve #DMSGPORT=80 -#-- Port for this application to serve http -#WEBPORT=8081 - #-- Local Port to serve over dmsg LOCALPORT=8086 From baaa2b8626c6c73f5eb811172443940486411beb Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 10:53:06 -0500 Subject: [PATCH 02/15] split up dmsgweb source code into multiple files ; make dmsgwebsrv able to proxy multiple ports over dmsg --- cmd/dmsgweb/commands/dmsgweb.go | 478 +---------------------------- cmd/dmsgweb/commands/dmsgwebsrv.go | 212 +++++++++++++ cmd/dmsgweb/commands/root.go | 351 +++++++++++++++++++++ 3 files changed, 564 insertions(+), 477 deletions(-) create mode 100644 cmd/dmsgweb/commands/dmsgwebsrv.go create mode 100644 cmd/dmsgweb/commands/root.go diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index e821b8db..c84a50f2 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -8,18 +8,13 @@ import ( "log" "net" "net/http" - "net/http/httputil" - "net/url" "os" "os/signal" "path/filepath" "regexp" "runtime" - "strconv" "strings" - "sync" "syscall" - "time" "github.com/bitfield/script" "github.com/confiant-inc/go-socks5" @@ -31,9 +26,6 @@ import ( "github.com/skycoin/skywire-utilities/pkg/skyenv" "github.com/spf13/cobra" "golang.org/x/net/proxy" - - "github.com/skycoin/dmsg/pkg/disc" - dmsg "github.com/skycoin/dmsg/pkg/dmsg" "github.com/skycoin/dmsg/pkg/dmsghttp" ) @@ -56,23 +48,6 @@ func (r *customResolver) Resolve(ctx context.Context, name string) (context.Cont return ctx, nil, nil } -var ( - httpC http.Client - dmsgDisc string - dmsgSessions int - filterDomainSuffix string - sk cipher.SecKey - pk cipher.PubKey - dmsgWebLog *logging.Logger - logLvl string - webPort uint - proxyPort uint - addProxy string - resolveDmsgAddr string - wg sync.WaitGroup - isEnvs bool -) - const dmsgwebenvname = "DMSGWEB" var dmsgwebconffile = os.Getenv(dmsgwebenvname) @@ -177,7 +152,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile <-ctx.Done() cancel() closeDmsg() - os.Exit(0) //this should not be necessary + os.Exit(0) }() httpC = http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgC)} @@ -305,436 +280,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile }, } -var ( - dmsgPort uint - dmsgSess int - wl string - wlkeys []cipher.PubKey - localPort uint - err error -) - -const dmsgwebsrvenvname = "DMSGWEBSRV" - -var dmsgwebsrvconffile = os.Getenv(dmsgwebsrvenvname) - -func init() { - RootCmd.AddCommand(srvCmd) - srvCmd.Flags().UintVarP(&localPort, "lport", "l", scriptExecUint("${LOCALPORT:-8086}", dmsgwebsrvconffile), "local application http interface port") - srvCmd.Flags().UintVarP(&dmsgPort, "dport", "d", scriptExecUint("${DMSGPORT:-80}", dmsgwebsrvconffile), "dmsg port to serve") - srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") - srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") - srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") - if os.Getenv("DMSGWEBSRV_SK") != "" { - sk.Set(os.Getenv("DMSGWEBSRV_SK")) //nolint - } - if scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile) != "" { - sk.Set(scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile)) //nolint - } - pk, _ = sk.PubKey() //nolint - srvCmd.Flags().VarP(&sk, "sk", "s", "a random key is generated if unspecified\n\r") - srvCmd.Flags().BoolVarP(&isEnvs, "envs", "z", false, "show example .conf file") - - srvCmd.CompletionOptions.DisableDefaultCmd = true -} - -var srvCmd = &cobra.Command{ - Use: "srv", - Short: "serve http from local port over dmsg", - Long: `DMSG web server - serve http interface from local port over dmsg` + func() string { - if _, err := os.Stat(dmsgwebsrvconffile); err == nil { - return ` - dmsenv file detected: ` + dmsgwebsrvconffile - } - return ` - .conf file may also be specified with - ` + dmsgwebsrvenvname + `=/path/to/dmsgwebsrv.conf skywire dmsg web srv` - }(), - Run: func(_ *cobra.Command, _ []string) { - if isEnvs { - envfile := srvenvfileLinux - if runtime.GOOS == "windows" { - envfileslice, _ := script.Echo(envfile).Slice() //nolint - for i := range envfileslice { - efs, _ := script.Echo(envfileslice[i]).Reject("##").Reject("#-").Reject("# ").Replace("#", "#$").String() //nolint - if efs != "" && efs != "\n" { - envfileslice[i] = strings.ReplaceAll(efs, "\n", "") - } - } - envfile = strings.Join(envfileslice, "\n") - } - fmt.Println(envfile) - os.Exit(0) - } - server() - }, -} - -func server() { - log := logging.MustGetLogger("dmsgwebsrv") - - ctx, cancel := cmdutil.SignalContext(context.Background(), log) - defer cancel() - pk, err = sk.PubKey() - if err != nil { - pk, sk = cipher.GenerateKeyPair() - } - if wl != "" { - wlk := strings.Split(wl, ",") - for _, key := range wlk { - var pk0 cipher.PubKey - err := pk0.Set(key) - if err == nil { - wlkeys = append(wlkeys, pk0) - } - } - } - if len(wlkeys) > 0 { - if len(wlkeys) == 1 { - log.Info(fmt.Sprintf("%d key whitelisted", len(wlkeys))) - } else { - log.Info(fmt.Sprintf("%d keys whitelisted", len(wlkeys))) - } - } - - dmsgC := dmsg.NewClient(pk, sk, disc.NewHTTP(dmsgDisc, &http.Client{}, log), dmsg.DefaultConfig()) - defer func() { - if err := dmsgC.Close(); err != nil { - log.WithError(err).Error() - } - }() - - go dmsgC.Serve(context.Background()) - - select { - case <-ctx.Done(): - log.WithError(ctx.Err()).Warn() - return - - case <-dmsgC.Ready(): - } - - lis, err := dmsgC.Listen(uint16(dmsgPort)) - if err != nil { - log.WithError(err).Fatal() - } - go func() { - <-ctx.Done() - if err := lis.Close(); err != nil { - log.WithError(err).Error() - } - }() - - r1 := gin.New() - r1.Use(gin.Recovery()) - r1.Use(loggingMiddleware()) - - authRoute := r1.Group("/") - if len(wlkeys) > 0 { - authRoute.Use(whitelistAuth(wlkeys)) - } - authRoute.Any("/*path", func(c *gin.Context) { - targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint - proxy := httputil.ReverseProxy{ - Director: func(req *http.Request) { - req.URL = targetURL - req.Host = targetURL.Host - req.Method = c.Request.Method - }, - Transport: &http.Transport{}, - } - proxy.ServeHTTP(c.Writer, c.Request) - }) - - serve := &http.Server{ - Handler: &ginHandler{Router: r1}, - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - - wg := new(sync.WaitGroup) - wg.Add(1) - go func() { - log.WithField("dmsg_addr", lis.Addr().String()).Info("Serving... ") - if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { - log.Fatalf("Serve1: %v", err) - } - wg.Done() - }() - - wg.Wait() -} - -func whitelistAuth(whitelistedPKs []cipher.PubKey) gin.HandlerFunc { - return func(c *gin.Context) { - remotePK, _, err := net.SplitHostPort(c.Request.RemoteAddr) - if err != nil { - c.Writer.WriteHeader(http.StatusInternalServerError) - c.Writer.Write([]byte("500 Internal Server Error")) //nolint - c.AbortWithStatus(http.StatusInternalServerError) - return - } - whitelisted := false - if len(whitelistedPKs) == 0 { - whitelisted = true - } else { - for _, whitelistedPK := range whitelistedPKs { - if remotePK == whitelistedPK.String() { - whitelisted = true - break - } - } - } - if whitelisted { - c.Next() - } else { - c.Writer.WriteHeader(http.StatusUnauthorized) - c.Writer.Write([]byte("401 Unauthorized")) //nolint - c.AbortWithStatus(http.StatusUnauthorized) - return - } - } -} - -type ginHandler struct { - Router *gin.Engine -} - -func (h *ginHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - h.Router.ServeHTTP(w, r) -} - -func startDmsg(ctx context.Context, pk cipher.PubKey, sk cipher.SecKey) (dmsgC *dmsg.Client, stop func(), err error) { - dmsgC = dmsg.NewClient(pk, sk, disc.NewHTTP(dmsgDisc, &http.Client{}, dmsgWebLog), &dmsg.Config{MinSessions: dmsgSessions}) - go dmsgC.Serve(context.Background()) - - stop = func() { - err := dmsgC.Close() - dmsgWebLog.WithError(err).Debug("Disconnected from dmsg network.") - fmt.Printf("\n") - } - dmsgWebLog.WithField("public_key", pk.String()).WithField("dmsg_disc", dmsgDisc). - Debug("Connecting to dmsg network...") - - select { - case <-ctx.Done(): - stop() - os.Exit(0) - return nil, nil, ctx.Err() - - case <-dmsgC.Ready(): - dmsgWebLog.Debug("Dmsg network ready.") - return dmsgC, stop, nil - } -} - -func loggingMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - start := time.Now() - c.Next() - latency := time.Since(start) - if latency > time.Minute { - latency = latency.Truncate(time.Second) - } - statusCode := c.Writer.Status() - method := c.Request.Method - path := c.Request.URL.Path - // Get the background color based on the status code - statusCodeBackgroundColor := getBackgroundColor(statusCode) - // Get the method color - methodColor := getMethodColor(method) - // Print the logging in a custom format which includes the publickeyfrom c.Request.RemoteAddr ex.: - // [DMSGHTTP] 2023/05/18 - 19:43:15 | 200 | 10.80885ms | | 02b5ee5333aa6b7f5fc623b7d5f35f505cb7f974e98a70751cf41962f84c8c4637:49153 | GET /node-info.json - fmt.Printf("[DMSGWEB] %s |%s %3d %s| %13v | %15s | %72s |%s %-7s %s %s\n", - time.Now().Format("2006/01/02 - 15:04:05"), - statusCodeBackgroundColor, - statusCode, - resetColor(), - latency, - c.ClientIP(), - c.Request.RemoteAddr, - methodColor, - method, - resetColor(), - path, - ) - } -} -func getBackgroundColor(statusCode int) string { - switch { - case statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices: - return green - case statusCode >= http.StatusMultipleChoices && statusCode < http.StatusBadRequest: - return white - case statusCode >= http.StatusBadRequest && statusCode < http.StatusInternalServerError: - return yellow - default: - return red - } -} - -func getMethodColor(method string) string { - switch method { - case http.MethodGet: - return blue - case http.MethodPost: - return cyan - case http.MethodPut: - return yellow - case http.MethodDelete: - return red - case http.MethodPatch: - return green - case http.MethodHead: - return magenta - case http.MethodOptions: - return white - default: - return reset - } -} - -func resetColor() string { - return reset -} - -const ( - green = "\033[97;42m" - white = "\033[90;47m" - yellow = "\033[90;43m" - red = "\033[97;41m" - blue = "\033[97;44m" - magenta = "\033[97;45m" - cyan = "\033[97;46m" - reset = "\033[0m" -) - -// Execute executes root CLI command. -func Execute() { - if err := RootCmd.Execute(); err != nil { - log.Fatal("Failed to execute command: ", err) - } -} - -func scriptExecString(s, envfile string) string { - if runtime.GOOS == "windows" { - var variable, defaultvalue string - if strings.Contains(s, ":-") { - parts := strings.SplitN(s, ":-", 2) - variable = parts[0] + "}" - defaultvalue = strings.TrimRight(parts[1], "}") - } else { - variable = s - defaultvalue = "" - } - out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() - if err == nil { - if (out == "") || (out == variable) { - return defaultvalue - } - return strings.TrimRight(out, "\n") - } - return defaultvalue - } - z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() - if err == nil { - return strings.TrimSpace(z) - } - return "" -} - -func scriptExecArray(s, envfile string) string { - if runtime.GOOS == "windows" { - variable := s - if strings.Contains(variable, "[@]}") { - variable = strings.TrimRight(variable, "[@]}") - variable = strings.TrimRight(variable, "{") - } - out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() - if err == nil { - if len(out) != 0 { - return "" - } - return strings.Join(out, ",") - } - } - y, err := script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() - if err == nil { - return strings.Join(y, ",") - } - return "" -} - -func scriptExecInt(s, envfile string) int { - if runtime.GOOS == "windows" { - var variable string - if strings.Contains(s, ":-") { - parts := strings.SplitN(s, ":-", 2) - variable = parts[0] + "}" - } else { - variable = s - } - out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() - if err == nil { - if (out == "") || (out == variable) { - return 0 - } - i, err := strconv.Atoi(strings.TrimSpace(strings.TrimRight(out, "\n"))) - if err == nil { - return i - } - return 0 - } - return 0 - } - z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() - if err == nil { - if z == "" { - return 0 - } - i, err := strconv.Atoi(z) - if err == nil { - return i - } - } - return 0 -} -func scriptExecUint(s, envfile string) uint { - if runtime.GOOS == "windows" { - var variable string - if strings.Contains(s, ":-") { - parts := strings.SplitN(s, ":-", 2) - variable = parts[0] + "}" - } else { - variable = s - } - out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() - if err == nil { - if (out == "") || (out == variable) { - return 0 - } - i, err := strconv.Atoi(strings.TrimSpace(strings.TrimRight(out, "\n"))) - if err == nil { - return uint(i) - } - return 0 - } - return 0 - } - z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() - if err == nil { - if z == "" { - return 0 - } - i, err := strconv.Atoi(z) - if err == nil { - return uint(i) - } - } - return uint(0) -} const envfileLinux = ` ######################################################################### @@ -764,25 +310,3 @@ const envfileLinux = ` #-- Set secret key #DMSGWEB_SK='' ` -const srvenvfileLinux = ` -######################################################################### -#-- DMSGWEB SRV CONFIG TEMPLATE -#-- Defaults shown -#-- Uncomment to change default value -######################################################################### - -#-- DMSG port to serve -#DMSGPORT=80 - -#-- Local Port to serve over dmsg -LOCALPORT=8086 - -#-- Number of dmsg servers to connect to (0 unlimits) -#DMSGSESSIONS=1 - -#-- Set secret key -#DMSGWEBSRV_SK='' - -#-- Whitelisted keys to access the web interface -#WHITELISTPKS=('') -` diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go new file mode 100644 index 00000000..d8cc5e6f --- /dev/null +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -0,0 +1,212 @@ +// Package commands cmd/dmsgweb/commands/dmsgweb.go +package commands + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "runtime" + "strings" + "sync" + "time" + + "github.com/bitfield/script" + "github.com/gin-gonic/gin" + "github.com/skycoin/skywire-utilities/pkg/cipher" + "github.com/skycoin/skywire-utilities/pkg/cmdutil" + "github.com/skycoin/skywire-utilities/pkg/logging" + "github.com/skycoin/skywire-utilities/pkg/skyenv" + "github.com/spf13/cobra" + + "github.com/skycoin/dmsg/pkg/disc" + dmsg "github.com/skycoin/dmsg/pkg/dmsg" +) + +const dmsgwebsrvenvname = "DMSGWEBSRV" + +var dmsgwebsrvconffile = os.Getenv(dmsgwebsrvenvname) + +func init() { + RootCmd.AddCommand(srvCmd) + srvCmd.Flags().UintSliceVarP(&localPort, "lport", "l", scriptExecUintSlice("${LOCALPORT[@]:-8086}", dmsgwebsrvconffile), "local application http interface port") + srvCmd.Flags().UintSliceVarP(&dmsgPort, "dport", "d", scriptExecUintSlice("${DMSGPORT[@]:-80}", dmsgwebsrvconffile), "dmsg port to serve") + srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") + srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") + srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") + if os.Getenv("DMSGWEBSRV_SK") != "" { + sk.Set(os.Getenv("DMSGWEBSRV_SK")) //nolint + } + if scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile) != "" { + sk.Set(scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile)) //nolint + } + pk, _ = sk.PubKey() //nolint + srvCmd.Flags().VarP(&sk, "sk", "s", "a random key is generated if unspecified\n\r") + srvCmd.Flags().BoolVarP(&isEnvs, "envs", "z", false, "show example .conf file") + + srvCmd.CompletionOptions.DisableDefaultCmd = true +} + +var srvCmd = &cobra.Command{ + Use: "srv", + Short: "serve http from local port over dmsg", + Long: `DMSG web server - serve http interface from local port over dmsg` + func() string { + if _, err := os.Stat(dmsgwebsrvconffile); err == nil { + return ` + dmsenv file detected: ` + dmsgwebsrvconffile + } + return ` + .conf file may also be specified with + ` + dmsgwebsrvenvname + `=/path/to/dmsgwebsrv.conf skywire dmsg web srv` + }(), + Run: func(_ *cobra.Command, _ []string) { + if isEnvs { + envfile := srvenvfileLinux + if runtime.GOOS == "windows" { + envfileslice, _ := script.Echo(envfile).Slice() //nolint + for i := range envfileslice { + efs, _ := script.Echo(envfileslice[i]).Reject("##").Reject("#-").Reject("# ").Replace("#", "#$").String() //nolint + if efs != "" && efs != "\n" { + envfileslice[i] = strings.ReplaceAll(efs, "\n", "") + } + } + envfile = strings.Join(envfileslice, "\n") + } + fmt.Println(envfile) + os.Exit(0) + } + server() + }, +} + +func server() { + log := logging.MustGetLogger("dmsgwebsrv") + + ctx, cancel := cmdutil.SignalContext(context.Background(), log) + + defer cancel() + pk, err = sk.PubKey() + if err != nil { + pk, sk = cipher.GenerateKeyPair() + } + if wl != "" { + wlk := strings.Split(wl, ",") + for _, key := range wlk { + var pk0 cipher.PubKey + err := pk0.Set(key) + if err == nil { + wlkeys = append(wlkeys, pk0) + } + } + } + if len(wlkeys) > 0 { + if len(wlkeys) == 1 { + log.Info(fmt.Sprintf("%d key whitelisted", len(wlkeys))) + } else { + log.Info(fmt.Sprintf("%d keys whitelisted", len(wlkeys))) + } + } + + dmsgC := dmsg.NewClient(pk, sk, disc.NewHTTP(dmsgDisc, &http.Client{}, log), dmsg.DefaultConfig()) + defer func() { + if err := dmsgC.Close(); err != nil { + log.WithError(err).Error() + } + }() + + go dmsgC.Serve(context.Background()) + + select { + case <-ctx.Done(): + log.WithError(ctx.Err()).Warn() + return + + case <-dmsgC.Ready(): + } + + var listN []net.Listener + + for _, dport := range dmsgPort { + lis, err := dmsgC.Listen(uint16(dport)) + if err != nil { + log.Fatalf("Error listening on port %d: %v", dport, err) + } + + listN = append(listN, lis) + + go func(l net.Listener) { + <-ctx.Done() + if err := l.Close(); err != nil { + log.Printf("Error closing listener on port %d: %v", dport, err) + log.WithError(err).Error() + } + }(lis) + } + + wg := new(sync.WaitGroup) + + for i, lpt := range localPort { + wg.Add(1) + go func(localPort uint, lis net.Listener) { + defer wg.Done() + r1 := gin.New() + r1.Use(gin.Recovery()) + r1.Use(loggingMiddleware()) + + authRoute := r1.Group("/") + if len(wlkeys) > 0 { + authRoute.Use(whitelistAuth(wlkeys)) + } + authRoute.Any("/*path", func(c *gin.Context) { + targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL = targetURL + req.Host = targetURL.Host + req.Method = c.Request.Method + }, + Transport: &http.Transport{}, + } + proxy.ServeHTTP(c.Writer, c.Request) + }) + serve := &http.Server{ + Handler: &ginHandler{Router: r1}, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + log.Printf("Serving on local port %v with DMSG listener %s", localPort, lis.Addr().String()) + if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { + log.Fatalf("Serve: %v", err) + } + }(lpt, listN[i]) + } + + wg.Wait() +} + +const srvenvfileLinux = ` +######################################################################### +#-- DMSGWEB SRV CONFIG TEMPLATE +#-- Defaults shown +#-- Uncomment to change default value +######################################################################### + +#-- DMSG port to serve +#DMSGPORT=80 + +#-- Local Port to serve over dmsg +LOCALPORT=8086 + +#-- Number of dmsg servers to connect to (0 unlimits) +#DMSGSESSIONS=1 + +#-- Set secret key +#DMSGWEBSRV_SK='' + +#-- Whitelisted keys to access the web interface +#WHITELISTPKS=('') +` diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go new file mode 100644 index 00000000..5417ef09 --- /dev/null +++ b/cmd/dmsgweb/commands/root.go @@ -0,0 +1,351 @@ +// Package commands cmd/dmsgweb/commands/dmsgweb.go +package commands + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "os" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/bitfield/script" + "github.com/gin-gonic/gin" + "github.com/skycoin/skywire-utilities/pkg/cipher" + "github.com/skycoin/skywire-utilities/pkg/logging" + + "github.com/skycoin/dmsg/pkg/disc" + dmsg "github.com/skycoin/dmsg/pkg/dmsg" +) + + +var ( + httpC http.Client + dmsgDisc string + dmsgSessions int + filterDomainSuffix string + sk cipher.SecKey + pk cipher.PubKey + dmsgWebLog *logging.Logger + logLvl string + webPort uint + proxyPort uint + addProxy string + resolveDmsgAddr string + wg sync.WaitGroup + isEnvs bool + dmsgPort []uint + dmsgSess int + wl string + wlkeys []cipher.PubKey + localPort []uint + err error +) + +// Execute executes root CLI command. +func Execute() { + if err := RootCmd.Execute(); err != nil { + log.Fatal("Failed to execute command: ", err) + } +} + +func startDmsg(ctx context.Context, pk cipher.PubKey, sk cipher.SecKey) (dmsgC *dmsg.Client, stop func(), err error) { + dmsgC = dmsg.NewClient(pk, sk, disc.NewHTTP(dmsgDisc, &http.Client{}, dmsgWebLog), &dmsg.Config{MinSessions: dmsgSessions}) + go dmsgC.Serve(context.Background()) + + stop = func() { + err := dmsgC.Close() + dmsgWebLog.WithError(err).Debug("Disconnected from dmsg network.") + fmt.Printf("\n") + } + dmsgWebLog.WithField("public_key", pk.String()).WithField("dmsg_disc", dmsgDisc). + Debug("Connecting to dmsg network...") + + select { + case <-ctx.Done(): + stop() + os.Exit(0) + return nil, nil, ctx.Err() + + case <-dmsgC.Ready(): + dmsgWebLog.Debug("Dmsg network ready.") + return dmsgC, stop, nil + } +} + +func scriptExecString(s, envfile string) string { + if runtime.GOOS == "windows" { + var variable, defaultvalue string + if strings.Contains(s, ":-") { + parts := strings.SplitN(s, ":-", 2) + variable = parts[0] + "}" + defaultvalue = strings.TrimRight(parts[1], "}") + } else { + variable = s + defaultvalue = "" + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() + if err == nil { + if (out == "") || (out == variable) { + return defaultvalue + } + return strings.TrimRight(out, "\n") + } + return defaultvalue + } + z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() + if err == nil { + return strings.TrimSpace(z) + } + return "" +} + +func scriptExecArray(s, envfile string) string { + if runtime.GOOS == "windows" { + variable := s + if strings.Contains(variable, "[@]}") { + variable = strings.TrimRight(variable, "[@]}") + variable = strings.TrimRight(variable, "{") + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() + if err == nil { + if len(out) != 0 { + return "" + } + return strings.Join(out, ",") + } + } + y, err := script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() + if err == nil { + return strings.Join(y, ",") + } + return "" +} + +func scriptExecUintSlice(s, envfile string) []uint { + var out []string + var err error + + if runtime.GOOS == "windows" { + variable := s + if strings.Contains(variable, "[@]}") { + variable = strings.TrimRight(variable, "[@]}") + variable = strings.TrimRight(variable, "{") + } + out, err = script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() + } else { + out, err = script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() + } + + if err != nil { + return []uint{} + } + + var res []uint + for _, item := range out { + num, err := strconv.ParseUint(item, 10, 64) + if err == nil { + res = append(res, uint(num)) + } + } + + return res +} + +func scriptExecInt(s, envfile string) int { + if runtime.GOOS == "windows" { + var variable string + if strings.Contains(s, ":-") { + parts := strings.SplitN(s, ":-", 2) + variable = parts[0] + "}" + } else { + variable = s + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() + if err == nil { + if (out == "") || (out == variable) { + return 0 + } + i, err := strconv.Atoi(strings.TrimSpace(strings.TrimRight(out, "\n"))) + if err == nil { + return i + } + return 0 + } + return 0 + } + z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() + if err == nil { + if z == "" { + return 0 + } + i, err := strconv.Atoi(z) + if err == nil { + return i + } + } + return 0 +} +func scriptExecUint(s, envfile string) uint { + if runtime.GOOS == "windows" { + var variable string + if strings.Contains(s, ":-") { + parts := strings.SplitN(s, ":-", 2) + variable = parts[0] + "}" + } else { + variable = s + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; echo %s"`, envfile, variable)).String() + if err == nil { + if (out == "") || (out == variable) { + return 0 + } + i, err := strconv.Atoi(strings.TrimSpace(strings.TrimRight(out, "\n"))) + if err == nil { + return uint(i) + } + return 0 + } + return 0 + } + z, err := script.Exec(fmt.Sprintf(`sh -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; printf "%s"'`, envfile, s)).String() + if err == nil { + if z == "" { + return 0 + } + i, err := strconv.Atoi(z) + if err == nil { + return uint(i) + } + } + return uint(0) +} + + + + +func whitelistAuth(whitelistedPKs []cipher.PubKey) gin.HandlerFunc { + return func(c *gin.Context) { + remotePK, _, err := net.SplitHostPort(c.Request.RemoteAddr) + if err != nil { + c.Writer.WriteHeader(http.StatusInternalServerError) + c.Writer.Write([]byte("500 Internal Server Error")) //nolint + c.AbortWithStatus(http.StatusInternalServerError) + return + } + whitelisted := false + if len(whitelistedPKs) == 0 { + whitelisted = true + } else { + for _, whitelistedPK := range whitelistedPKs { + if remotePK == whitelistedPK.String() { + whitelisted = true + break + } + } + } + if whitelisted { + c.Next() + } else { + c.Writer.WriteHeader(http.StatusUnauthorized) + c.Writer.Write([]byte("401 Unauthorized")) //nolint + c.AbortWithStatus(http.StatusUnauthorized) + return + } + } +} + +type ginHandler struct { + Router *gin.Engine +} + +func (h *ginHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.Router.ServeHTTP(w, r) +} + + +func loggingMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + latency := time.Since(start) + if latency > time.Minute { + latency = latency.Truncate(time.Second) + } + statusCode := c.Writer.Status() + method := c.Request.Method + path := c.Request.URL.Path + // Get the background color based on the status code + statusCodeBackgroundColor := getBackgroundColor(statusCode) + // Get the method color + methodColor := getMethodColor(method) + // Print the logging in a custom format which includes the publickeyfrom c.Request.RemoteAddr ex.: + // [DMSGHTTP] 2023/05/18 - 19:43:15 | 200 | 10.80885ms | | 02b5ee5333aa6b7f5fc623b7d5f35f505cb7f974e98a70751cf41962f84c8c4637:49153 | GET /node-info.json + fmt.Printf("[DMSGWEB] %s |%s %3d %s| %13v | %15s | %72s |%s %-7s %s %s\n", + time.Now().Format("2006/01/02 - 15:04:05"), + statusCodeBackgroundColor, + statusCode, + resetColor(), + latency, + c.ClientIP(), + c.Request.RemoteAddr, + methodColor, + method, + resetColor(), + path, + ) + } +} +func getBackgroundColor(statusCode int) string { + switch { + case statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices: + return green + case statusCode >= http.StatusMultipleChoices && statusCode < http.StatusBadRequest: + return white + case statusCode >= http.StatusBadRequest && statusCode < http.StatusInternalServerError: + return yellow + default: + return red + } +} + +func getMethodColor(method string) string { + switch method { + case http.MethodGet: + return blue + case http.MethodPost: + return cyan + case http.MethodPut: + return yellow + case http.MethodDelete: + return red + case http.MethodPatch: + return green + case http.MethodHead: + return magenta + case http.MethodOptions: + return white + default: + return reset + } +} + +func resetColor() string { + return reset +} + +const ( + green = "\033[97;42m" + white = "\033[90;47m" + yellow = "\033[90;43m" + red = "\033[97;41m" + blue = "\033[97;44m" + magenta = "\033[97;45m" + cyan = "\033[97;46m" + reset = "\033[0m" +) From 1f7a1fac2d2b641772dab4781bd4397c98a93572 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 10:57:17 -0500 Subject: [PATCH 03/15] log fatal error if same number of dmsg ports and local ports are not specified --- cmd/dmsgweb/commands/dmsgwebsrv.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index d8cc5e6f..00e37eac 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -32,8 +32,8 @@ var dmsgwebsrvconffile = os.Getenv(dmsgwebsrvenvname) func init() { RootCmd.AddCommand(srvCmd) - srvCmd.Flags().UintSliceVarP(&localPort, "lport", "l", scriptExecUintSlice("${LOCALPORT[@]:-8086}", dmsgwebsrvconffile), "local application http interface port") - srvCmd.Flags().UintSliceVarP(&dmsgPort, "dport", "d", scriptExecUintSlice("${DMSGPORT[@]:-80}", dmsgwebsrvconffile), "dmsg port to serve") + srvCmd.Flags().UintSliceVarP(&localPort, "lport", "l", scriptExecUintSlice("${LOCALPORT[@]:-8086}", dmsgwebsrvconffile), "local application http interface port(s)") + srvCmd.Flags().UintSliceVarP(&dmsgPort, "dport", "d", scriptExecUintSlice("${DMSGPORT[@]:-80}", dmsgwebsrvconffile), "dmsg port(s) to serve") srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") @@ -84,6 +84,9 @@ var srvCmd = &cobra.Command{ func server() { log := logging.MustGetLogger("dmsgwebsrv") + if len(localPort) != len(dmsgPort) { + log.Fatal(fmt.Sprintf("the same number of local ports as dmsg ports must be specified ; local ports: %v ; dmsg ports: %v", len(localPort), len(dmsgPort))) + } ctx, cancel := cmdutil.SignalContext(context.Background(), log) From cb42f1502931bfc3e8515afd3c3eaf0f98a18a16 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 11:15:26 -0500 Subject: [PATCH 04/15] minor text formatting changes --- cmd/dmsgweb/commands/dmsgwebsrv.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 00e37eac..2b7e9700 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -181,7 +181,7 @@ func server() { ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } - log.Printf("Serving on local port %v with DMSG listener %s", localPort, lis.Addr().String()) + log.Printf("Serving on dmsg port %v with DMSG listener %s", localPort, lis.Addr().String()) if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { log.Fatalf("Serve: %v", err) } @@ -199,10 +199,10 @@ const srvenvfileLinux = ` ######################################################################### #-- DMSG port to serve -#DMSGPORT=80 +#DMSGPORT=('80') #-- Local Port to serve over dmsg -LOCALPORT=8086 +LOCALPORT=('8086') #-- Number of dmsg servers to connect to (0 unlimits) #DMSGSESSIONS=1 From 0903d25de23c77fea88e8794294a4ea0524a8095 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 11:17:16 -0500 Subject: [PATCH 05/15] make format --- cmd/dmsgweb/commands/dmsgweb.go | 3 +- cmd/dmsgweb/commands/dmsgwebsrv.go | 76 +++++++++++++++--------------- cmd/dmsgweb/commands/root.go | 17 +++---- pkg/dmsg/const.go | 4 +- pkg/dmsgctrl/serve_listener.go | 4 +- pkg/dmsgpty/ui_windows.go | 4 +- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index c84a50f2..a4397f3b 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -26,6 +26,7 @@ import ( "github.com/skycoin/skywire-utilities/pkg/skyenv" "github.com/spf13/cobra" "golang.org/x/net/proxy" + "github.com/skycoin/dmsg/pkg/dmsghttp" ) @@ -280,8 +281,6 @@ dmsgweb conf file detected: ` + dmsgwebconffile }, } - - const envfileLinux = ` ######################################################################### #-- DMSGWEB CONFIG TEMPLATE diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 2b7e9700..128061c2 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -149,46 +149,46 @@ func server() { }(lis) } - wg := new(sync.WaitGroup) - - for i, lpt := range localPort { - wg.Add(1) - go func(localPort uint, lis net.Listener) { - defer wg.Done() - r1 := gin.New() - r1.Use(gin.Recovery()) - r1.Use(loggingMiddleware()) - - authRoute := r1.Group("/") - if len(wlkeys) > 0 { - authRoute.Use(whitelistAuth(wlkeys)) - } - authRoute.Any("/*path", func(c *gin.Context) { - targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint - proxy := httputil.ReverseProxy{ - Director: func(req *http.Request) { - req.URL = targetURL - req.Host = targetURL.Host - req.Method = c.Request.Method - }, - Transport: &http.Transport{}, - } - proxy.ServeHTTP(c.Writer, c.Request) - }) - serve := &http.Server{ - Handler: &ginHandler{Router: r1}, - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - log.Printf("Serving on dmsg port %v with DMSG listener %s", localPort, lis.Addr().String()) - if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { - log.Fatalf("Serve: %v", err) + wg := new(sync.WaitGroup) + + for i, lpt := range localPort { + wg.Add(1) + go func(localPort uint, lis net.Listener) { + defer wg.Done() + r1 := gin.New() + r1.Use(gin.Recovery()) + r1.Use(loggingMiddleware()) + + authRoute := r1.Group("/") + if len(wlkeys) > 0 { + authRoute.Use(whitelistAuth(wlkeys)) + } + authRoute.Any("/*path", func(c *gin.Context) { + targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL = targetURL + req.Host = targetURL.Host + req.Method = c.Request.Method + }, + Transport: &http.Transport{}, } - }(lpt, listN[i]) - } + proxy.ServeHTTP(c.Writer, c.Request) + }) + serve := &http.Server{ + Handler: &ginHandler{Router: r1}, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + log.Printf("Serving on dmsg port %v with DMSG listener %s", localPort, lis.Addr().String()) + if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { + log.Fatalf("Serve: %v", err) + } + }(lpt, listN[i]) + } - wg.Wait() + wg.Wait() } const srvenvfileLinux = ` diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index 5417ef09..abab0420 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -23,7 +23,6 @@ import ( dmsg "github.com/skycoin/dmsg/pkg/dmsg" ) - var ( httpC http.Client dmsgDisc string @@ -39,12 +38,12 @@ var ( resolveDmsgAddr string wg sync.WaitGroup isEnvs bool - dmsgPort []uint - dmsgSess int - wl string - wlkeys []cipher.PubKey - localPort []uint - err error + dmsgPort []uint + dmsgSess int + wl string + wlkeys []cipher.PubKey + localPort []uint + err error ) // Execute executes root CLI command. @@ -226,9 +225,6 @@ func scriptExecUint(s, envfile string) uint { return uint(0) } - - - func whitelistAuth(whitelistedPKs []cipher.PubKey) gin.HandlerFunc { return func(c *gin.Context) { remotePK, _, err := net.SplitHostPort(c.Request.RemoteAddr) @@ -268,7 +264,6 @@ func (h *ginHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.Router.ServeHTTP(w, r) } - func loggingMiddleware() gin.HandlerFunc { return func(c *gin.Context) { start := time.Now() diff --git a/pkg/dmsg/const.go b/pkg/dmsg/const.go index cac2c8b8..731c2bc8 100644 --- a/pkg/dmsg/const.go +++ b/pkg/dmsg/const.go @@ -1,7 +1,9 @@ // Package dmsg pkg/dmsg/const.go package dmsg -import "time" +import ( + "time" +) // Constants. const ( diff --git a/pkg/dmsgctrl/serve_listener.go b/pkg/dmsgctrl/serve_listener.go index da172181..2fd9b928 100644 --- a/pkg/dmsgctrl/serve_listener.go +++ b/pkg/dmsgctrl/serve_listener.go @@ -1,7 +1,9 @@ // Package dmsgctrl pkg/dmsgctrl/serve_listener.go package dmsgctrl -import "net" +import ( + "net" +) // ServeListener serves a listener with dmsgctrl.Control. // It returns a channel for incoming Controls. diff --git a/pkg/dmsgpty/ui_windows.go b/pkg/dmsgpty/ui_windows.go index 915f553e..2cb3d978 100644 --- a/pkg/dmsgpty/ui_windows.go +++ b/pkg/dmsgpty/ui_windows.go @@ -4,7 +4,9 @@ // Package dmsgpty pkg/dmsgpty/ui_windows.go package dmsgpty -import "golang.org/x/sys/windows" +import ( + "golang.org/x/sys/windows" +) func (ui *UI) uiStartSize(ptyC *PtyClient) error { ws, err := NewWinSize(&windows.Coord{ From 6bbbfbaedda002282145b9c3798c56a55d528e1a Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 11:27:20 -0500 Subject: [PATCH 06/15] fix CI error --- cmd/dmsgweb/commands/dmsgwebsrv.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 128061c2..a0fe0984 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -133,20 +133,21 @@ func server() { var listN []net.Listener for _, dport := range dmsgPort { - lis, err := dmsgC.Listen(uint16(dport)) - if err != nil { - log.Fatalf("Error listening on port %d: %v", dport, err) - } - - listN = append(listN, lis) - - go func(l net.Listener) { - <-ctx.Done() - if err := l.Close(); err != nil { - log.Printf("Error closing listener on port %d: %v", dport, err) - log.WithError(err).Error() - } - }(lis) + lis, err := dmsgC.Listen(uint16(dport)) + if err != nil { + log.Fatalf("Error listening on port %d: %v", dport, err) + } + + listN = append(listN, lis) + + dport := dport + go func(l net.Listener, port uint) { + <-ctx.Done() + if err := l.Close(); err != nil { + log.Printf("Error closing listener on port %d: %v", port, err) + log.WithError(err).Error() + } + }(lis, dport) } wg := new(sync.WaitGroup) From fc512359e0c8abb6a941c817ea384f67ecb767c5 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 11:42:15 -0500 Subject: [PATCH 07/15] make format ; update Makefile --- Makefile | 2 +- cmd/dmsgweb/commands/dmsgwebsrv.go | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index c06f404c..7a7b754c 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ install-linters-windows: ## Install linters on windows ${OPTS} go install github.com/incu6us/goimports-reviser@latest format: ## Formats the code. Must have goimports and goimports-reviser installed (use make install-linters). - ${OPTS} goimports -local ${DMSG_REPO} -w . + ${OPTS} goimports -w -local ${DMSG_REPO} ./pkg ./cmd ./internal ./examples find . -type f -name '*.go' -not -path "./.git/*" -not -path "./vendor/*" -exec goimports-reviser -project-name ${DMSG_REPO} {} \; diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index a0fe0984..524c862a 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -133,21 +133,21 @@ func server() { var listN []net.Listener for _, dport := range dmsgPort { - lis, err := dmsgC.Listen(uint16(dport)) - if err != nil { - log.Fatalf("Error listening on port %d: %v", dport, err) - } - - listN = append(listN, lis) - - dport := dport - go func(l net.Listener, port uint) { - <-ctx.Done() - if err := l.Close(); err != nil { - log.Printf("Error closing listener on port %d: %v", port, err) - log.WithError(err).Error() - } - }(lis, dport) + lis, err := dmsgC.Listen(uint16(dport)) + if err != nil { + log.Fatalf("Error listening on port %d: %v", dport, err) + } + + listN = append(listN, lis) + + dport := dport + go func(l net.Listener, port uint) { + <-ctx.Done() + if err := l.Close(); err != nil { + log.Printf("Error closing listener on port %d: %v", port, err) + log.WithError(err).Error() + } + }(lis, dport) } wg := new(sync.WaitGroup) From f0fd1b5a5152ed2ea2b3845dce70cc7be3a4b881 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 18:48:40 -0500 Subject: [PATCH 08/15] handle raw tcp connections --- cmd/dmsgweb/commands/dmsgweb.go | 188 +++++++++++++++++++++-------- cmd/dmsgweb/commands/dmsgwebsrv.go | 110 ++++++++++++----- cmd/dmsgweb/commands/root.go | 3 + 3 files changed, 217 insertions(+), 84 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index a4397f3b..228a29f7 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -12,6 +12,7 @@ import ( "os/signal" "path/filepath" "regexp" + "strconv" "runtime" "strings" "syscall" @@ -28,6 +29,8 @@ import ( "golang.org/x/net/proxy" "github.com/skycoin/dmsg/pkg/dmsghttp" + dmsg "github.com/skycoin/dmsg/pkg/dmsg" + ) type customResolver struct{} @@ -62,6 +65,8 @@ func init() { RootCmd.Flags().StringVarP(&resolveDmsgAddr, "resolve", "t", scriptExecString("${RESOLVEPK}", dmsgwebconffile), "resolve the specified dmsg address:port on the local port & disable proxy") RootCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "d", skyenv.DmsgDiscAddr, "dmsg discovery url") RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebconffile), "number of dmsg servers to connect to") + RootCmd.Flags().BoolVarP(&rawTCP, "raw-tcp", "c", false, "proxy local application as raw TCP") // New flag + RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m") if os.Getenv("DMSGWEB_SK") != "" { sk.Set(os.Getenv("DMSGWEB_SK")) //nolint @@ -114,6 +119,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile fmt.Println(envfile) os.Exit(0) } + c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) //nolint go func() { @@ -142,7 +148,12 @@ dmsgweb conf file detected: ` + dmsgwebconffile if err != nil { pk, sk = cipher.GenerateKeyPair() } - + dmsgWebLog.Info("dmsg client pk: %v", pk.String()) + dmsgAddr = strings.Split(resolveDmsgAddr, ":") + err = dialPK.Set(dmsgAddr[0]) + if err != nil { + log.Fatalf("failed to parse dmsg
: : %v", err) + } dmsgC, closeDmsg, err := startDmsg(ctx, pk, sk) if err != nil { dmsgWebLog.WithError(err).Fatal("failed to start dmsg") @@ -210,75 +221,146 @@ dmsgweb conf file detected: ` + dmsgwebconffile dmsgWebLog.Debug("Stopped serving SOCKS5 proxy on " + socksAddr) }() } - r := gin.New() - r.Use(gin.Recovery()) + if !rawTCP { + proxyHTTPConn(localPort[0], dmsgWebLog) + } else { + proxyTCPConn(localPort[0], dmsgC, dmsgWebLog) + } + wg.Wait() + }, +} +func proxyHTTPConn(localPort uint, log *logging.Logger) { + r := gin.New() + + r.Use(gin.Recovery()) - r.Use(loggingMiddleware()) + r.Use(loggingMiddleware()) - r.Any("/*path", func(c *gin.Context) { - var urlStr string - if resolveDmsgAddr != "" { - urlStr = fmt.Sprintf("dmsg://%s%s", resolveDmsgAddr, c.Param("path")) - if c.Request.URL.RawQuery != "" { - urlStr = fmt.Sprintf("%s?%s", urlStr, c.Request.URL.RawQuery) - } + r.Any("/*path", func(c *gin.Context) { + var urlStr string + if resolveDmsgAddr != "" { + urlStr = fmt.Sprintf("dmsg://%s%s", resolveDmsgAddr, c.Param("path")) + if c.Request.URL.RawQuery != "" { + urlStr = fmt.Sprintf("%s?%s", urlStr, c.Request.URL.RawQuery) + } + } else { + hostParts := strings.Split(c.Request.Host, ":") + var dmsgp string + if len(hostParts) > 1 { + dmsgp = hostParts[1] } else { - hostParts := strings.Split(c.Request.Host, ":") - var dmsgp string - if len(hostParts) > 1 { - dmsgp = hostParts[1] - } else { - dmsgp = "80" - } - urlStr = fmt.Sprintf("dmsg://%s:%s%s", strings.TrimRight(hostParts[0], filterDomainSuffix), dmsgp, c.Param("path")) - if c.Request.URL.RawQuery != "" { - urlStr = fmt.Sprintf("%s?%s", urlStr, c.Request.URL.RawQuery) - } + dmsgp = "80" + } + urlStr = fmt.Sprintf("dmsg://%s:%s%s", strings.TrimRight(hostParts[0], filterDomainSuffix), dmsgp, c.Param("path")) + if c.Request.URL.RawQuery != "" { + urlStr = fmt.Sprintf("%s?%s", urlStr, c.Request.URL.RawQuery) } + } - fmt.Printf("Proxying request: %s %s\n", c.Request.Method, urlStr) - req, err := http.NewRequest(c.Request.Method, urlStr, c.Request.Body) - if err != nil { - c.String(http.StatusInternalServerError, "Failed to create HTTP request") - return + fmt.Printf("Proxying request: %s %s\n", c.Request.Method, urlStr) + req, err := http.NewRequest(c.Request.Method, urlStr, c.Request.Body) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to create HTTP request") + return + } + + for header, values := range c.Request.Header { + for _, value := range values { + req.Header.Add(header, value) } + } - for header, values := range c.Request.Header { - for _, value := range values { - req.Header.Add(header, value) + resp, err := httpC.Do(req) + if err != nil { + c.String(http.StatusInternalServerError, "Failed to connect to HTTP server") + fmt.Printf("Error: %v\n", err) + return + } + defer resp.Body.Close() //nolint + + for header, values := range resp.Header { + for _, value := range values { + c.Writer.Header().Add(header, value) + } + } + + c.Status(resp.StatusCode) + if _, err := io.Copy(c.Writer, resp.Body); err != nil { + c.String(http.StatusInternalServerError, "Failed to copy response body") + fmt.Printf("Error copying response body: %v\n", err) + } + }) + wg.Add(1) + go func() { + dmsgWebLog.Debug(fmt.Sprintf("Serving http on: http://127.0.0.1:%v", webPort)) + r.Run(":" + fmt.Sprintf("%v", webPort)) //nolint + dmsgWebLog.Debug(fmt.Sprintf("Stopped serving http on: http://127.0.0.1:%v", webPort)) + wg.Done() + }() +} +func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { + // Start listening on the specified local port + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", webPort)) + if err != nil { + log.Fatalf("Failed to start TCP listener on port %d: %v", webPort, err) + } + defer listener.Close() + log.Printf("Serving TCP on 127.0.0.1:%d", webPort) + + // Accept incoming TCP connections and proxy them + for { + conn, err := listener.Accept() + if err != nil { + log.Printf("Failed to accept connection: %v", err) + continue + } + + wg.Add(1) + go func(conn net.Conn) { + defer wg.Done() + defer conn.Close() + + var dialPort uint64 + if len(dmsgAddr) > 1 { + dialPort, err = strconv.ParseUint(dmsgAddr[1], 10, 64) + if err != nil { + log.Fatalf("Failed to parse dmsg port: %v", err) + return } + } else { + dialPort = uint64(80) } - resp, err := httpC.Do(req) + // Dial the dmsg address using the dmsg client + dmsgConn, err := dmsgC.DialStream(context.Background(), dmsg.Addr{PK: dialPK, Port: uint16(dialPort)}) if err != nil { - c.String(http.StatusInternalServerError, "Failed to connect to HTTP server") - fmt.Printf("Error: %v\n", err) + log.Printf("Failed to dial dmsg address %s: %v", resolveDmsgAddr, err) return } - defer resp.Body.Close() //nolint + defer dmsgConn.Close() - for header, values := range resp.Header { - for _, value := range values { - c.Writer.Header().Add(header, value) + // Proxy data from TCP connection to dmsg connection + go func() { + _, err := io.Copy(dmsgConn, conn) + if err != nil { + log.Printf("Error copying data to dmsg server: %v", err) } - } + // Close dmsgConn after copying is done + dmsgConn.Close() + }() - c.Status(resp.StatusCode) - if _, err := io.Copy(c.Writer, resp.Body); err != nil { - c.String(http.StatusInternalServerError, "Failed to copy response body") - fmt.Printf("Error copying response body: %v\n", err) - } - }) - wg.Add(1) - go func() { - dmsgWebLog.Debug(fmt.Sprintf("Serving http on: http://127.0.0.1:%v", webPort)) - r.Run(":" + fmt.Sprintf("%v", webPort)) //nolint - dmsgWebLog.Debug(fmt.Sprintf("Stopped serving http on: http://127.0.0.1:%v", webPort)) - wg.Done() - }() - wg.Wait() - }, + // Proxy data from dmsg connection to TCP connection + go func() { + _, err := io.Copy(conn, dmsgConn) + if err != nil { + log.Printf("Error copying data from dmsg server: %v", err) + } + // Close conn after copying is done + conn.Close() + }() + }(conn) + } } const envfileLinux = ` diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 524c862a..617a3caa 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -4,6 +4,7 @@ package commands import ( "context" "fmt" + "io" "net" "net/http" "net/http/httputil" @@ -37,6 +38,7 @@ func init() { srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") + srvCmd.Flags().BoolVarP(&rawTCP, "raw-tcp", "c", false, "proxy local application as raw TCP") // New flag if os.Getenv("DMSGWEBSRV_SK") != "" { sk.Set(os.Getenv("DMSGWEBSRV_SK")) //nolint } @@ -52,8 +54,8 @@ func init() { var srvCmd = &cobra.Command{ Use: "srv", - Short: "serve http from local port over dmsg", - Long: `DMSG web server - serve http interface from local port over dmsg` + func() string { + Short: "serve http or raw TCP from local port over dmsg", + Long: `DMSG web server - serve http or raw TCP interface from local port over dmsg` + func() string { if _, err := os.Stat(dmsgwebsrvconffile); err == nil { return ` dmsenv file detected: ` + dmsgwebsrvconffile @@ -95,6 +97,8 @@ func server() { if err != nil { pk, sk = cipher.GenerateKeyPair() } + log.Infof("dmsg client pk: %v", pk.String()) + if wl != "" { wlk := strings.Split(wl, ",") for _, key := range wlk { @@ -156,35 +160,10 @@ func server() { wg.Add(1) go func(localPort uint, lis net.Listener) { defer wg.Done() - r1 := gin.New() - r1.Use(gin.Recovery()) - r1.Use(loggingMiddleware()) - - authRoute := r1.Group("/") - if len(wlkeys) > 0 { - authRoute.Use(whitelistAuth(wlkeys)) - } - authRoute.Any("/*path", func(c *gin.Context) { - targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint - proxy := httputil.ReverseProxy{ - Director: func(req *http.Request) { - req.URL = targetURL - req.Host = targetURL.Host - req.Method = c.Request.Method - }, - Transport: &http.Transport{}, - } - proxy.ServeHTTP(c.Writer, c.Request) - }) - serve := &http.Server{ - Handler: &ginHandler{Router: r1}, - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - log.Printf("Serving on dmsg port %v with DMSG listener %s", localPort, lis.Addr().String()) - if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { - log.Fatalf("Serve: %v", err) + if rawTCP { + proxyTCPConnections(localPort, lis, log) + } else { + proxyHTTPConnections(localPort, lis, log) } }(lpt, listN[i]) } @@ -192,6 +171,72 @@ func server() { wg.Wait() } +func proxyHTTPConnections(localPort uint, lis net.Listener, log *logging.Logger) { + r1 := gin.New() + r1.Use(gin.Recovery()) + r1.Use(loggingMiddleware()) + + authRoute := r1.Group("/") + if len(wlkeys) > 0 { + authRoute.Use(whitelistAuth(wlkeys)) + } + authRoute.Any("/*path", func(c *gin.Context) { + targetURL, _ := url.Parse(fmt.Sprintf("http://127.0.0.1:%v%s?%s", localPort, c.Request.URL.Path, c.Request.URL.RawQuery)) //nolint + proxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + req.URL = targetURL + req.Host = targetURL.Host + req.Method = c.Request.Method + }, + Transport: &http.Transport{}, + } + proxy.ServeHTTP(c.Writer, c.Request) + }) + serve := &http.Server{ + Handler: &ginHandler{Router: r1}, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + log.Printf("Serving HTTP on dmsg port %v with DMSG listener %s", localPort, lis.Addr().String()) + if err := serve.Serve(lis); err != nil && err != http.ErrServerClosed { + log.Fatalf("Serve: %v", err) + } +} + +func proxyTCPConnections(localPort uint, lis net.Listener, log *logging.Logger) { + for { + conn, err := lis.Accept() + if err != nil { + log.Printf("Error accepting connection: %v", err) + return + } + + go handleTCPConnection(conn, localPort, log) + } +} + +func handleTCPConnection(dmsgConn net.Conn, localPort uint, log *logging.Logger) { + defer dmsgConn.Close() + + localConn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", localPort)) + if err != nil { + log.Printf("Error connecting to local port %d: %v", localPort, err) + return + } + defer localConn.Close() + + copyConn := func(dst net.Conn, src net.Conn) { + _, err := io.Copy(dst, src) + if err != nil { + log.Printf("Error during copy: %v", err) + } + } + + go copyConn(dmsgConn, localConn) + go copyConn(localConn, dmsgConn) +} + const srvenvfileLinux = ` ######################################################################### #-- DMSGWEB SRV CONFIG TEMPLATE @@ -213,4 +258,7 @@ LOCALPORT=('8086') #-- Whitelisted keys to access the web interface #WHITELISTPKS=('') + +#-- Proxy as raw TCP +#RAW_TCP=false ` diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index abab0420..16d51fed 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -27,6 +27,8 @@ var ( httpC http.Client dmsgDisc string dmsgSessions int + dmsgAddr []string + dialPK cipher.PubKey filterDomainSuffix string sk cipher.SecKey pk cipher.PubKey @@ -44,6 +46,7 @@ var ( wlkeys []cipher.PubKey localPort []uint err error + rawTCP bool ) // Execute executes root CLI command. From 2764434ee6c615f2e586f9e5a5b28a4495723a3b Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Thu, 13 Jun 2024 19:20:53 -0500 Subject: [PATCH 09/15] framework for handling raw tcp & udp --- cmd/dmsgweb/commands/dmsgweb.go | 73 +++++++++++++++++++++++++++--- cmd/dmsgweb/commands/dmsgwebsrv.go | 59 +++++++++++++++++++++++- cmd/dmsgweb/commands/root.go | 7 +-- 3 files changed, 127 insertions(+), 12 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index 228a29f7..1e212dde 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -12,8 +12,8 @@ import ( "os/signal" "path/filepath" "regexp" - "strconv" "runtime" + "strconv" "strings" "syscall" @@ -28,9 +28,8 @@ import ( "github.com/spf13/cobra" "golang.org/x/net/proxy" - "github.com/skycoin/dmsg/pkg/dmsghttp" dmsg "github.com/skycoin/dmsg/pkg/dmsg" - + "github.com/skycoin/dmsg/pkg/dmsghttp" ) type customResolver struct{} @@ -65,7 +64,8 @@ func init() { RootCmd.Flags().StringVarP(&resolveDmsgAddr, "resolve", "t", scriptExecString("${RESOLVEPK}", dmsgwebconffile), "resolve the specified dmsg address:port on the local port & disable proxy") RootCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "d", skyenv.DmsgDiscAddr, "dmsg discovery url") RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebconffile), "number of dmsg servers to connect to") - RootCmd.Flags().BoolVarP(&rawTCP, "raw-tcp", "c", false, "proxy local application as raw TCP") // New flag + RootCmd.Flags().BoolVarP(&rawTCP, "rt", "c", false, "proxy local port as raw TCP") // New flag + RootCmd.Flags().BoolVarP(&rawUDP, "ru", "u", false, "proxy local port as raw UDP") // New flag RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m") if os.Getenv("DMSGWEB_SK") != "" { @@ -134,6 +134,9 @@ dmsgweb conf file detected: ` + dmsgwebconffile logging.SetLevel(lvl) } } + if rawTCP && rawUDP { + log.Fatal("must specify either --rt or --ru flags not both") + } if filterDomainSuffix == "" { dmsgWebLog.Fatal("domain suffix to filter cannot be an empty string") @@ -222,14 +225,19 @@ dmsgweb conf file detected: ` + dmsgwebconffile }() } - if !rawTCP { - proxyHTTPConn(localPort[0], dmsgWebLog) - } else { + if rawTCP { proxyTCPConn(localPort[0], dmsgC, dmsgWebLog) } + // if rawUDP { + // proxyUDPConn(localPort[0], dmsgC, dmsgWebLog) + // } + if !rawTCP && !rawUDP { + proxyHTTPConn(localPort[0], dmsgWebLog) + } wg.Wait() }, } + func proxyHTTPConn(localPort uint, log *logging.Logger) { r := gin.New() @@ -363,6 +371,57 @@ func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { } } +/* + func proxyUDPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { + // Resolve the dmsg address + var dialPort uint64 + if len(dmsgAddr) > 1 { + dialPort, _ = strconv.ParseUint(dmsgAddr[1], 10, 64) + } else { + dialPort = 80 + } + + // Listen for incoming UDP packets on the specified port + addr := fmt.Sprintf(":%d", webPort) + conn, err := net.ListenPacket("udp", addr) + if err != nil { + log.Fatalf("Failed to listen on UDP port %d: %v", webPort, err) + } + defer conn.Close() + log.Printf("Serving UDP on 127.0.0.1:%d", webPort) + + // Buffer to hold incoming UDP data + buffer := make([]byte, 65535) // Maximum UDP packet size + + for { + // Read UDP packet from the connection + n, addr, err := conn.ReadFrom(buffer) + if err != nil { + log.Printf("Error reading UDP packet: %v", err) + continue + } + + // Destination dmsg address + dmsgAddr := dmsg.Addr{PK: dialPK, Port: uint16(dialPort)} + + // Dial dmsg connection + dmsgConn, err := dmsgC.DialPacketConn(context.Background(), dmsgAddr) + if err != nil { + log.Printf("Failed to dial dmsg address %s: %v", dmsgAddr.String(), err) + continue + } + + // Write UDP packet to dmsg connection + _, err = dmsgConn.WriteTo(buffer[:n], addr) + if err != nil { + log.Printf("Error writing UDP packet to dmsg server: %v", err) + } + + // Close dmsg connection + dmsgConn.Close() + } + } +*/ const envfileLinux = ` ######################################################################### #-- DMSGWEB CONFIG TEMPLATE diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 617a3caa..706e1acc 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -38,7 +38,9 @@ func init() { srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") - srvCmd.Flags().BoolVarP(&rawTCP, "raw-tcp", "c", false, "proxy local application as raw TCP") // New flag + srvCmd.Flags().BoolVarP(&rawTCP, "rt", "c", false, "proxy local port as raw TCP") // New flag + srvCmd.Flags().BoolVarP(&rawUDP, "ru", "u", false, "proxy local port as raw UDP") // New flag + if os.Getenv("DMSGWEBSRV_SK") != "" { sk.Set(os.Getenv("DMSGWEBSRV_SK")) //nolint } @@ -80,6 +82,7 @@ var srvCmd = &cobra.Command{ fmt.Println(envfile) os.Exit(0) } + server() }, } @@ -89,6 +92,9 @@ func server() { if len(localPort) != len(dmsgPort) { log.Fatal(fmt.Sprintf("the same number of local ports as dmsg ports must be specified ; local ports: %v ; dmsg ports: %v", len(localPort), len(dmsgPort))) } + if rawTCP && rawUDP { + log.Fatal("must specify either --rt or --ru flags not both") + } ctx, cancel := cmdutil.SignalContext(context.Background(), log) @@ -162,7 +168,11 @@ func server() { defer wg.Done() if rawTCP { proxyTCPConnections(localPort, lis, log) - } else { + } + // if rawUDP { + // handleUDPConnection(localPort, lis, log) + // } + if !rawTCP && !rawUDP { proxyHTTPConnections(localPort, lis, log) } }(lpt, listN[i]) @@ -237,6 +247,51 @@ func handleTCPConnection(dmsgConn net.Conn, localPort uint, log *logging.Logger) go copyConn(localConn, dmsgConn) } +/* +func handleUDPConnection(localPort uint, conn net.PacketConn, dmsgC *dmsg.Client, log *logging.Logger) { + buffer := make([]byte, 65535) + + for { + n, addr, err := conn.ReadFrom(buffer) + if err != nil { + log.Printf("Error reading UDP packet: %v", err) + continue + } + + err = dmsgC.SendUDP(buffer[:n], localPort, addr) + if err != nil { + log.Printf("Error sending UDP packet via dmsg client: %v", err) + continue + } + + responseBuffer := make([]byte, 65535) + n, _, err = dmsgC.ReceiveUDP(responseBuffer) + if err != nil { + log.Printf("Error receiving UDP response from dmsg client: %v", err) + continue + } + + _, err = conn.WriteTo(responseBuffer[:n], addr) + if err != nil { + log.Printf("Error sending UDP response to client: %v", err) + continue + } + } +} + +func proxyUDPConnections(conn net.PacketConn, data []byte, addr net.Addr, webPort uint, log *logging.Logger) { + for { + conn, err := lis.Accept() + if err != nil { + log.Printf("Error accepting connection: %v", err) + return + } + + go handleUDPConnection(conn, localPort, log) + } +} +*/ + const srvenvfileLinux = ` ######################################################################### #-- DMSGWEB SRV CONFIG TEMPLATE diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index 16d51fed..7b06fb7a 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -27,8 +27,8 @@ var ( httpC http.Client dmsgDisc string dmsgSessions int - dmsgAddr []string - dialPK cipher.PubKey + dmsgAddr []string + dialPK cipher.PubKey filterDomainSuffix string sk cipher.SecKey pk cipher.PubKey @@ -46,7 +46,8 @@ var ( wlkeys []cipher.PubKey localPort []uint err error - rawTCP bool + rawTCP bool + rawUDP bool ) // Execute executes root CLI command. From fbe08b1d955cb40c42b1b0622d21dea5d09883a3 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Fri, 14 Jun 2024 10:54:23 -0500 Subject: [PATCH 10/15] fix ci errors --- cmd/dmsgweb/commands/dmsgweb.go | 20 ++++++++++---------- cmd/dmsgweb/commands/dmsgwebsrv.go | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index 1e212dde..cb748c2b 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -226,19 +226,19 @@ dmsgweb conf file detected: ` + dmsgwebconffile } if rawTCP { - proxyTCPConn(localPort[0], dmsgC, dmsgWebLog) + proxyTCPConn(dmsgC, dmsgWebLog) } // if rawUDP { - // proxyUDPConn(localPort[0], dmsgC, dmsgWebLog) + // proxyUDPConn(dmsgC, dmsgWebLog) // } if !rawTCP && !rawUDP { - proxyHTTPConn(localPort[0], dmsgWebLog) + proxyHTTPConn(dmsgWebLog) } wg.Wait() }, } -func proxyHTTPConn(localPort uint, log *logging.Logger) { +func proxyHTTPConn(log *logging.Logger) { r := gin.New() r.Use(gin.Recovery()) @@ -307,13 +307,13 @@ func proxyHTTPConn(localPort uint, log *logging.Logger) { wg.Done() }() } -func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { +func proxyTCPConn(dmsgC *dmsg.Client, log *logging.Logger) { // Start listening on the specified local port listener, err := net.Listen("tcp", fmt.Sprintf(":%d", webPort)) if err != nil { log.Fatalf("Failed to start TCP listener on port %d: %v", webPort, err) } - defer listener.Close() + defer listener.Close() //nolint log.Printf("Serving TCP on 127.0.0.1:%d", webPort) // Accept incoming TCP connections and proxy them @@ -327,7 +327,7 @@ func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { wg.Add(1) go func(conn net.Conn) { defer wg.Done() - defer conn.Close() + defer conn.Close() //nolint var dialPort uint64 if len(dmsgAddr) > 1 { @@ -346,7 +346,7 @@ func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { log.Printf("Failed to dial dmsg address %s: %v", resolveDmsgAddr, err) return } - defer dmsgConn.Close() + defer dmsgConn.Close() //nolint // Proxy data from TCP connection to dmsg connection go func() { @@ -355,7 +355,7 @@ func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { log.Printf("Error copying data to dmsg server: %v", err) } // Close dmsgConn after copying is done - dmsgConn.Close() + dmsgConn.Close() //nolint }() // Proxy data from dmsg connection to TCP connection @@ -365,7 +365,7 @@ func proxyTCPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { log.Printf("Error copying data from dmsg server: %v", err) } // Close conn after copying is done - conn.Close() + conn.Close() //nolint }() }(conn) } diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 706e1acc..f154da30 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -227,14 +227,14 @@ func proxyTCPConnections(localPort uint, lis net.Listener, log *logging.Logger) } func handleTCPConnection(dmsgConn net.Conn, localPort uint, log *logging.Logger) { - defer dmsgConn.Close() + defer dmsgConn.Close() //nolint localConn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", localPort)) if err != nil { log.Printf("Error connecting to local port %d: %v", localPort, err) return } - defer localConn.Close() + defer localConn.Close() //nolint copyConn := func(dst net.Conn, src net.Conn) { _, err := io.Copy(dst, src) From 278dc5a2945859197fc3937ffde95f0f9ec5875a Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Fri, 14 Jun 2024 10:59:39 -0500 Subject: [PATCH 11/15] fix ci errors --- cmd/dmsgweb/commands/dmsgweb.go | 15 ++++++++------- cmd/dmsgweb/commands/root.go | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index cb748c2b..10b8df64 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -226,19 +226,20 @@ dmsgweb conf file detected: ` + dmsgwebconffile } if rawTCP { - proxyTCPConn(dmsgC, dmsgWebLog) + proxyTCPConn() + } + if rawUDP { + log.Fatalf("handling raw udp not yet implemented") +// proxyUDPConn() } - // if rawUDP { - // proxyUDPConn(dmsgC, dmsgWebLog) - // } if !rawTCP && !rawUDP { - proxyHTTPConn(dmsgWebLog) + proxyHTTPConn() } wg.Wait() }, } -func proxyHTTPConn(log *logging.Logger) { +func proxyHTTPConn() { r := gin.New() r.Use(gin.Recovery()) @@ -307,7 +308,7 @@ func proxyHTTPConn(log *logging.Logger) { wg.Done() }() } -func proxyTCPConn(dmsgC *dmsg.Client, log *logging.Logger) { +func proxyTCPConn() { // Start listening on the specified local port listener, err := net.Listen("tcp", fmt.Sprintf(":%d", webPort)) if err != nil { diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index 7b06fb7a..c042d5aa 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -25,6 +25,7 @@ import ( var ( httpC http.Client + dmsgC *dmsg.Client dmsgDisc string dmsgSessions int dmsgAddr []string From f5bb5e1291fcf0f490ed176f9488f7265bb0280b Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Fri, 14 Jun 2024 11:08:08 -0500 Subject: [PATCH 12/15] make format --- cmd/dmsgweb/commands/dmsgweb.go | 2 +- cmd/dmsgweb/commands/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index 10b8df64..fa69237b 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -230,7 +230,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile } if rawUDP { log.Fatalf("handling raw udp not yet implemented") -// proxyUDPConn() + // proxyUDPConn() } if !rawTCP && !rawUDP { proxyHTTPConn() diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index c042d5aa..a2b20bfd 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -25,7 +25,7 @@ import ( var ( httpC http.Client - dmsgC *dmsg.Client + dmsgC *dmsg.Client dmsgDisc string dmsgSessions int dmsgAddr []string From 6075e023858720c5b2f4c7f44520b977c8859783 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Tue, 18 Jun 2024 10:35:24 -0500 Subject: [PATCH 13/15] make dmsgweb work with multiple ports --- cmd/dmsgweb/commands/dmsgweb.go | 202 +++++++++++++---------------- cmd/dmsgweb/commands/dmsgwebsrv.go | 100 +++++--------- cmd/dmsgweb/commands/root.go | 75 +++++++++-- 3 files changed, 184 insertions(+), 193 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index fa69237b..f3460793 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -60,19 +60,17 @@ func init() { RootCmd.Flags().StringVarP(&filterDomainSuffix, "filter", "f", ".dmsg", "domain suffix to filter") RootCmd.Flags().UintVarP(&proxyPort, "socks", "q", scriptExecUint("${PROXYPORT:-4445}", dmsgwebconffile), "port to serve the socks5 proxy") RootCmd.Flags().StringVarP(&addProxy, "proxy", "r", scriptExecString("${ADDPROXY}", dmsgwebconffile), "configure additional socks5 proxy for dmsgweb (i.e. 127.0.0.1:1080)") - RootCmd.Flags().UintVarP(&webPort, "port", "p", scriptExecUint("${WEBPORT:-8080}", dmsgwebconffile), "port to serve the web application") - RootCmd.Flags().StringVarP(&resolveDmsgAddr, "resolve", "t", scriptExecString("${RESOLVEPK}", dmsgwebconffile), "resolve the specified dmsg address:port on the local port & disable proxy") + RootCmd.Flags().UintSliceVarP(&webPort, "port", "p", scriptExecUintSlice("${WEBPORT[@]:-8080}", dmsgwebconffile), "port(s) to serve the web application") + RootCmd.Flags().StringSliceVarP(&resolveDmsgAddr, "resolve", "t", scriptExecStringSlice("${RESOLVEPK[@]}", dmsgwebconffile), "resolve the specified dmsg address:port on the local port & disable proxy") RootCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "d", skyenv.DmsgDiscAddr, "dmsg discovery url") RootCmd.Flags().IntVarP(&dmsgSessions, "sess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebconffile), "number of dmsg servers to connect to") - RootCmd.Flags().BoolVarP(&rawTCP, "rt", "c", false, "proxy local port as raw TCP") // New flag - RootCmd.Flags().BoolVarP(&rawUDP, "ru", "u", false, "proxy local port as raw UDP") // New flag - + RootCmd.Flags().BoolSliceVarP(&rawTCP, "rt", "c", scriptExecBoolSlice("${RAWTCP[@]:-false}", dmsgwebconffile), "proxy local port as raw TCP") RootCmd.Flags().StringVarP(&logLvl, "loglvl", "l", "", "[ debug | warn | error | fatal | panic | trace | info ]\033[0m") - if os.Getenv("DMSGWEB_SK") != "" { - sk.Set(os.Getenv("DMSGWEB_SK")) //nolint + if os.Getenv("DMSGWEBSK") != "" { + sk.Set(os.Getenv("DMSGWEBSK")) //nolint } - if scriptExecString("${DMSGWEB_SK}", dmsgwebconffile) != "" { - sk.Set(scriptExecString("${DMSGWEB_SK}", dmsgwebconffile)) //nolint + if scriptExecString("${DMSGWEBSK}", dmsgwebconffile) != "" { + sk.Set(scriptExecString("${DMSGWEBSK}", dmsgwebconffile)) //nolint } RootCmd.Flags().VarP(&sk, "sk", "s", "a random key is generated if unspecified\n\r") RootCmd.Flags().BoolVarP(&isEnvs, "envs", "z", false, "show example .conf file") @@ -134,8 +132,36 @@ dmsgweb conf file detected: ` + dmsgwebconffile logging.SetLevel(lvl) } } - if rawTCP && rawUDP { - log.Fatal("must specify either --rt or --ru flags not both") + + if len(resolveDmsgAddr) > 0 && len(webPort) != len(resolveDmsgAddr) { + dmsgWebLog.Fatal("-resolve --t flag cannot contain a different number of elements than -port -p flag") + } + if len(resolveDmsgAddr) == 0 && len(webPort) > 1 { + dmsgWebLog.Fatal("-port --p flag cannot specify multiple ports without specifying multiple dmsg address:port(s) to -resolve --t flag") + } + + seenResolveDmsgAddr := make(map[string]bool) + for _, item := range resolveDmsgAddr { + if seenResolveDmsgAddr[item] { + dmsgWebLog.Fatal("-resolve --t flag cannot contain duplicates") + } + seenResolveDmsgAddr[item] = true + } + + seenWebPort := make(map[uint]bool) + for _, item := range webPort { + if seenWebPort[item] { + dmsgWebLog.Fatal("-port --p flag cannot contain duplicates") + } + seenWebPort[item] = true + } + + if len(rawTCP) < len(resolveDmsgAddr) { + for len(rawTCP) < len(resolveDmsgAddr) { + rawTCP = append(rawTCP, false) + } + } else if len(rawTCP) > len(resolveDmsgAddr) { + rawTCP = rawTCP[:len(resolveDmsgAddr)] } if filterDomainSuffix == "" { @@ -152,10 +178,24 @@ dmsgweb conf file detected: ` + dmsgwebconffile pk, sk = cipher.GenerateKeyPair() } dmsgWebLog.Info("dmsg client pk: %v", pk.String()) - dmsgAddr = strings.Split(resolveDmsgAddr, ":") - err = dialPK.Set(dmsgAddr[0]) - if err != nil { - log.Fatalf("failed to parse dmsg
: : %v", err) + + if len(resolveDmsgAddr) > 0 { + for i, dmsgaddr := range resolveDmsgAddr { + dmsgAddr = strings.Split(dmsgaddr, ":") + err = dialPK[i].Set(dmsgAddr[0]) + if err != nil { + log.Fatalf("failed to parse dmsg
: : %v", err) + } + if len(dmsgAddr) > 1 { + dport, err := strconv.ParseUint(dmsgAddr[1], 10, 64) + if err != nil { + log.Fatalf("Failed to parse dmsg port: %v", err) + } + dmsgPorts[i] = uint(dport) + } else { + dmsgPorts[i] = uint(80) + } + } } dmsgC, closeDmsg, err := startDmsg(ctx, pk, sk) if err != nil { @@ -172,7 +212,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile httpC = http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgC)} - if resolveDmsgAddr == "" { + if len(resolveDmsgAddr) == 0 { // Create a SOCKS5 server with custom name resolution conf := &socks5.Config{ Resolver: &customResolver{}, @@ -225,21 +265,26 @@ dmsgweb conf file detected: ` + dmsgwebconffile }() } - if rawTCP { - proxyTCPConn() - } - if rawUDP { - log.Fatalf("handling raw udp not yet implemented") - // proxyUDPConn() - } - if !rawTCP && !rawUDP { - proxyHTTPConn() + if len(resolveDmsgAddr) == 0 && len(webPort) == 1 { + if rawTCP[0] { + proxyTCPConn(-1) + } else { + proxyHTTPConn(-1) + } + } else { + for i, _ := range resolveDmsgAddr { + if rawTCP[i] { + proxyTCPConn(i) + } else { + proxyHTTPConn(i) + } + } } wg.Wait() }, } -func proxyHTTPConn() { +func proxyHTTPConn(n int) { r := gin.New() r.Use(gin.Recovery()) @@ -248,8 +293,8 @@ func proxyHTTPConn() { r.Any("/*path", func(c *gin.Context) { var urlStr string - if resolveDmsgAddr != "" { - urlStr = fmt.Sprintf("dmsg://%s%s", resolveDmsgAddr, c.Param("path")) + if n > -1 { + urlStr = fmt.Sprintf("dmsg://%s%s", resolveDmsgAddr[n], c.Param("path")) if c.Request.URL.RawQuery != "" { urlStr = fmt.Sprintf("%s?%s", urlStr, c.Request.URL.RawQuery) } @@ -308,16 +353,14 @@ func proxyHTTPConn() { wg.Done() }() } -func proxyTCPConn() { - // Start listening on the specified local port - listener, err := net.Listen("tcp", fmt.Sprintf(":%d", webPort)) +func proxyTCPConn(n int) { + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", webPort[n])) if err != nil { - log.Fatalf("Failed to start TCP listener on port %d: %v", webPort, err) + log.Fatalf("Failed to start TCP listener on port %d: %v", webPort[n], err) } defer listener.Close() //nolint - log.Printf("Serving TCP on 127.0.0.1:%d", webPort) + log.Printf("Serving TCP on 127.0.0.1:%d", webPort[n]) - // Accept incoming TCP connections and proxy them for { conn, err := listener.Accept() if err != nil { @@ -326,128 +369,65 @@ func proxyTCPConn() { } wg.Add(1) - go func(conn net.Conn) { + go func(conn net.Conn, n int) { defer wg.Done() defer conn.Close() //nolint - var dialPort uint64 - if len(dmsgAddr) > 1 { - dialPort, err = strconv.ParseUint(dmsgAddr[1], 10, 64) - if err != nil { - log.Fatalf("Failed to parse dmsg port: %v", err) - return - } - } else { - dialPort = uint64(80) - } - - // Dial the dmsg address using the dmsg client - dmsgConn, err := dmsgC.DialStream(context.Background(), dmsg.Addr{PK: dialPK, Port: uint16(dialPort)}) + dmsgConn, err := dmsgC.DialStream(context.Background(), dmsg.Addr{PK: dialPK[n], Port: uint16(dmsgPorts[n])}) if err != nil { - log.Printf("Failed to dial dmsg address %s: %v", resolveDmsgAddr, err) + log.Printf("Failed to dial dmsg address %v:%v %v", dialPK[n].String(), dmsgPorts[n], err) return } defer dmsgConn.Close() //nolint - // Proxy data from TCP connection to dmsg connection go func() { _, err := io.Copy(dmsgConn, conn) if err != nil { log.Printf("Error copying data to dmsg server: %v", err) } - // Close dmsgConn after copying is done dmsgConn.Close() //nolint }() - // Proxy data from dmsg connection to TCP connection go func() { _, err := io.Copy(conn, dmsgConn) if err != nil { log.Printf("Error copying data from dmsg server: %v", err) } - // Close conn after copying is done conn.Close() //nolint }() - }(conn) + }(conn, n) } } -/* - func proxyUDPConn(webPort uint, dmsgC *dmsg.Client, log *logging.Logger) { - // Resolve the dmsg address - var dialPort uint64 - if len(dmsgAddr) > 1 { - dialPort, _ = strconv.ParseUint(dmsgAddr[1], 10, 64) - } else { - dialPort = 80 - } - - // Listen for incoming UDP packets on the specified port - addr := fmt.Sprintf(":%d", webPort) - conn, err := net.ListenPacket("udp", addr) - if err != nil { - log.Fatalf("Failed to listen on UDP port %d: %v", webPort, err) - } - defer conn.Close() - log.Printf("Serving UDP on 127.0.0.1:%d", webPort) - - // Buffer to hold incoming UDP data - buffer := make([]byte, 65535) // Maximum UDP packet size - - for { - // Read UDP packet from the connection - n, addr, err := conn.ReadFrom(buffer) - if err != nil { - log.Printf("Error reading UDP packet: %v", err) - continue - } - - // Destination dmsg address - dmsgAddr := dmsg.Addr{PK: dialPK, Port: uint16(dialPort)} - - // Dial dmsg connection - dmsgConn, err := dmsgC.DialPacketConn(context.Background(), dmsgAddr) - if err != nil { - log.Printf("Failed to dial dmsg address %s: %v", dmsgAddr.String(), err) - continue - } - - // Write UDP packet to dmsg connection - _, err = dmsgConn.WriteTo(buffer[:n], addr) - if err != nil { - log.Printf("Error writing UDP packet to dmsg server: %v", err) - } - - // Close dmsg connection - dmsgConn.Close() - } - } -*/ const envfileLinux = ` ######################################################################### #-- DMSGWEB CONFIG TEMPLATE #-- Defaults shown #-- Uncomment to change default value +#-- WEBPORT and DMSGPORT must contain the same number of elements ######################################################################### #-- Set port for proxy interface #PROXYPORT=4445 -#-- Configure additional proxy for dmsgvlc to use +#-- Configure additional socks5 proxy for dmsgweb to use to connect to dmsg #ADDPROXY='127.0.0.1:1080' #-- Web Interface Port -#WEBPORT=8080 +#WEBPORT=('8080') #-- Resove a specific PK to the web port (also disables proxy) -#RESOLVEPK='' +#RESOLVEPK=('') + +#-- Use raw tcp mode instead of http (also disables proxy) +#RAWTCP=('false') #-- Number of dmsg servers to connect to (0 unlimits) -#DMSGSESSIONS=1 +#DMSGSESSIONS=2 #-- Dmsg port to use -#DMSGPORT=80 +#DMSGPORT=('80') #-- Set secret key -#DMSGWEB_SK='' +#DMSGWEBSK='' ` diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index f154da30..058be880 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -35,17 +35,15 @@ func init() { RootCmd.AddCommand(srvCmd) srvCmd.Flags().UintSliceVarP(&localPort, "lport", "l", scriptExecUintSlice("${LOCALPORT[@]:-8086}", dmsgwebsrvconffile), "local application http interface port(s)") srvCmd.Flags().UintSliceVarP(&dmsgPort, "dport", "d", scriptExecUintSlice("${DMSGPORT[@]:-80}", dmsgwebsrvconffile), "dmsg port(s) to serve") - srvCmd.Flags().StringVarP(&wl, "wl", "w", scriptExecArray("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") + srvCmd.Flags().StringSliceVarP(&wl, "wl", "w", scriptExecStringSlice("${WHITELISTPKS[@]}", dmsgwebsrvconffile), "whitelisted keys for dmsg authenticated routes\r") srvCmd.Flags().StringVarP(&dmsgDisc, "dmsg-disc", "D", skyenv.DmsgDiscAddr, "dmsg discovery url") srvCmd.Flags().IntVarP(&dmsgSess, "dsess", "e", scriptExecInt("${DMSGSESSIONS:-1}", dmsgwebsrvconffile), "dmsg sessions") - srvCmd.Flags().BoolVarP(&rawTCP, "rt", "c", false, "proxy local port as raw TCP") // New flag - srvCmd.Flags().BoolVarP(&rawUDP, "ru", "u", false, "proxy local port as raw UDP") // New flag - - if os.Getenv("DMSGWEBSRV_SK") != "" { - sk.Set(os.Getenv("DMSGWEBSRV_SK")) //nolint + srvCmd.Flags().BoolSliceVarP(&rawTCP, "rt", "c", scriptExecBoolSlice("${RAWTCP[@]:-false}", dmsgwebsrvconffile), "proxy local port as raw TCP") + if os.Getenv("DMSGWEBSRVSK") != "" { + sk.Set(os.Getenv("DMSGWEBSRVSK")) //nolint } - if scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile) != "" { - sk.Set(scriptExecString("${DMSGWEBSRV_SK}", dmsgwebsrvconffile)) //nolint + if scriptExecString("${DMSGWEBSRVSK}", dmsgwebsrvconffile) != "" { + sk.Set(scriptExecString("${DMSGWEBSRVSK}", dmsgwebsrvconffile)) //nolint } pk, _ = sk.PubKey() //nolint srvCmd.Flags().VarP(&sk, "sk", "s", "a random key is generated if unspecified\n\r") @@ -92,8 +90,21 @@ func server() { if len(localPort) != len(dmsgPort) { log.Fatal(fmt.Sprintf("the same number of local ports as dmsg ports must be specified ; local ports: %v ; dmsg ports: %v", len(localPort), len(dmsgPort))) } - if rawTCP && rawUDP { - log.Fatal("must specify either --rt or --ru flags not both") + + seenLocalPort := make(map[uint]bool) + for _, item := range localPort { + if seenLocalPort[item] { + log.Fatal("-lport --l flag cannot contain duplicates") + } + seenLocalPort[item] = true + } + + seenDmsgPort := make(map[uint]bool) + for _, item := range dmsgPort { + if seenDmsgPort[item] { + log.Fatal("-dport --d flag cannot contain duplicates") + } + seenDmsgPort[item] = true } ctx, cancel := cmdutil.SignalContext(context.Background(), log) @@ -105,9 +116,8 @@ func server() { } log.Infof("dmsg client pk: %v", pk.String()) - if wl != "" { - wlk := strings.Split(wl, ",") - for _, key := range wlk { + if len(wl) > 0 { + for _, key := range wl { var pk0 cipher.PubKey err := pk0.Set(key) if err == nil { @@ -164,18 +174,14 @@ func server() { for i, lpt := range localPort { wg.Add(1) - go func(localPort uint, lis net.Listener) { + go func(localPort uint, rtcp bool, lis net.Listener) { defer wg.Done() - if rawTCP { + if rtcp { proxyTCPConnections(localPort, lis, log) - } - // if rawUDP { - // handleUDPConnection(localPort, lis, log) - // } - if !rawTCP && !rawUDP { + } else { proxyHTTPConnections(localPort, lis, log) } - }(lpt, listN[i]) + }(lpt, rawTCP[i], listN[i]) } wg.Wait() @@ -247,73 +253,29 @@ func handleTCPConnection(dmsgConn net.Conn, localPort uint, log *logging.Logger) go copyConn(localConn, dmsgConn) } -/* -func handleUDPConnection(localPort uint, conn net.PacketConn, dmsgC *dmsg.Client, log *logging.Logger) { - buffer := make([]byte, 65535) - - for { - n, addr, err := conn.ReadFrom(buffer) - if err != nil { - log.Printf("Error reading UDP packet: %v", err) - continue - } - - err = dmsgC.SendUDP(buffer[:n], localPort, addr) - if err != nil { - log.Printf("Error sending UDP packet via dmsg client: %v", err) - continue - } - - responseBuffer := make([]byte, 65535) - n, _, err = dmsgC.ReceiveUDP(responseBuffer) - if err != nil { - log.Printf("Error receiving UDP response from dmsg client: %v", err) - continue - } - - _, err = conn.WriteTo(responseBuffer[:n], addr) - if err != nil { - log.Printf("Error sending UDP response to client: %v", err) - continue - } - } -} - -func proxyUDPConnections(conn net.PacketConn, data []byte, addr net.Addr, webPort uint, log *logging.Logger) { - for { - conn, err := lis.Accept() - if err != nil { - log.Printf("Error accepting connection: %v", err) - return - } - - go handleUDPConnection(conn, localPort, log) - } -} -*/ - const srvenvfileLinux = ` ######################################################################### #-- DMSGWEB SRV CONFIG TEMPLATE #-- Defaults shown #-- Uncomment to change default value +#-- LOCALPORT and DMSGPORT must contain the same number of elements ######################################################################### #-- DMSG port to serve #DMSGPORT=('80') #-- Local Port to serve over dmsg -LOCALPORT=('8086') +#LOCALPORT=('8086') #-- Number of dmsg servers to connect to (0 unlimits) #DMSGSESSIONS=1 #-- Set secret key -#DMSGWEBSRV_SK='' +#DMSGWEBSRVSK='' #-- Whitelisted keys to access the web interface #WHITELISTPKS=('') #-- Proxy as raw TCP -#RAW_TCP=false +#RAWTCP=('false') ` diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index a2b20bfd..e184fc25 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -29,26 +29,26 @@ var ( dmsgDisc string dmsgSessions int dmsgAddr []string - dialPK cipher.PubKey + dialPK []cipher.PubKey filterDomainSuffix string sk cipher.SecKey pk cipher.PubKey dmsgWebLog *logging.Logger logLvl string - webPort uint + webPort []uint proxyPort uint addProxy string - resolveDmsgAddr string + resolveDmsgAddr []string wg sync.WaitGroup isEnvs bool dmsgPort []uint + dmsgPorts []uint dmsgSess int - wl string + wl []string wlkeys []cipher.PubKey localPort []uint err error - rawTCP bool - rawUDP bool + rawTCP []bool ) // Execute executes root CLI command. @@ -109,7 +109,30 @@ func scriptExecString(s, envfile string) string { return "" } -func scriptExecArray(s, envfile string) string { +/* + func scriptExecArray(s, envfile string) string { + if runtime.GOOS == "windows" { + variable := s + if strings.Contains(variable, "[@]}") { + variable = strings.TrimRight(variable, "[@]}") + variable = strings.TrimRight(variable, "{") + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() + if err == nil { + if len(out) != 0 { + return "" + } + return strings.Join(out, ",") + } + } + y, err := script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() + if err == nil { + return strings.Join(y, ",") + } + return "" + } +*/ +func scriptExecStringSlice(s, envfile string) []string { if runtime.GOOS == "windows" { variable := s if strings.Contains(variable, "[@]}") { @@ -118,17 +141,43 @@ func scriptExecArray(s, envfile string) string { } out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() if err == nil { - if len(out) != 0 { - return "" - } - return strings.Join(out, ",") + return out } } y, err := script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() if err == nil { - return strings.Join(y, ",") + return y } - return "" + return []string{} +} + +func scriptExecBoolSlice(s, envfile string) []bool { + var result []bool + + if runtime.GOOS == "windows" { + variable := s + if strings.Contains(variable, "[@]}") { + variable = strings.TrimRight(variable, "[@]}") + variable = strings.TrimRight(variable, "{") + } + out, err := script.Exec(fmt.Sprintf(`powershell -c '$SKYENV = "%s"; if ($SKYENV -ne "" -and (Test-Path $SKYENV)) { . $SKYENV }; foreach ($item in %s) { Write-Host $item }'`, envfile, variable)).Slice() + if err == nil { + for _, item := range out { + result = append(result, item != "") + } + return result + } + } else { + y, err := script.Exec(fmt.Sprintf(`bash -c 'SKYENV=%s ; if [[ $SKYENV != "" ]] && [[ -f $SKYENV ]] ; then source $SKYENV ; fi ; for _i in %s ; do echo "$_i" ; done'`, envfile, s)).Slice() + if err == nil { + for _, item := range y { + result = append(result, item != "") + } + return result + } + } + + return result } func scriptExecUintSlice(s, envfile string) []uint { From 4d376f8a08f0a558e6b29f3b56f8376afdfa0654 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Tue, 18 Jun 2024 10:38:26 -0500 Subject: [PATCH 14/15] fix ci errors --- cmd/dmsgweb/commands/dmsgweb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/dmsgweb/commands/dmsgweb.go b/cmd/dmsgweb/commands/dmsgweb.go index f3460793..ca2ce14c 100644 --- a/cmd/dmsgweb/commands/dmsgweb.go +++ b/cmd/dmsgweb/commands/dmsgweb.go @@ -272,7 +272,7 @@ dmsgweb conf file detected: ` + dmsgwebconffile proxyHTTPConn(-1) } } else { - for i, _ := range resolveDmsgAddr { + for i := range resolveDmsgAddr { if rawTCP[i] { proxyTCPConn(i) } else { From f3c5d61f2b4ce7fd2d57437001888053c00b7407 Mon Sep 17 00:00:00 2001 From: Moses Narrow Date: Tue, 18 Jun 2024 13:32:45 -0500 Subject: [PATCH 15/15] fix package comments --- cmd/dmsgweb/commands/dmsgwebsrv.go | 2 +- cmd/dmsgweb/commands/root.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/dmsgweb/commands/dmsgwebsrv.go b/cmd/dmsgweb/commands/dmsgwebsrv.go index 058be880..547df639 100644 --- a/cmd/dmsgweb/commands/dmsgwebsrv.go +++ b/cmd/dmsgweb/commands/dmsgwebsrv.go @@ -1,4 +1,4 @@ -// Package commands cmd/dmsgweb/commands/dmsgweb.go +// Package commands cmd/dmsgweb/commands/dmsgwebsrv.go package commands import ( diff --git a/cmd/dmsgweb/commands/root.go b/cmd/dmsgweb/commands/root.go index e184fc25..37643db7 100644 --- a/cmd/dmsgweb/commands/root.go +++ b/cmd/dmsgweb/commands/root.go @@ -1,4 +1,4 @@ -// Package commands cmd/dmsgweb/commands/dmsgweb.go +// Package commands cmd/dmsgweb/commands/root.go package commands import (