diff --git a/addon/addon.go b/addon/addon.go deleted file mode 100644 index 3d5114d..0000000 --- a/addon/addon.go +++ /dev/null @@ -1,88 +0,0 @@ -package addon - -import ( - "time" - - "github.com/lqqyt2423/go-mitmproxy/connection" - "github.com/lqqyt2423/go-mitmproxy/flow" - _log "github.com/sirupsen/logrus" -) - -var log = _log.WithField("at", "addon") - -type Addon interface { - // A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests. - ClientConnected(*connection.Client) - - // A client connection has been closed (either by us or the client). - ClientDisconnected(*connection.Client) - - // Mitmproxy has connected to a server. - ServerConnected(*flow.ConnContext) - - // A server connection has been closed (either by us or the server). - ServerDisconnected(*flow.ConnContext) - - // HTTP request headers were successfully read. At this point, the body is empty. - Requestheaders(*flow.Flow) - - // The full HTTP request has been read. - Request(*flow.Flow) - - // HTTP response headers were successfully read. At this point, the body is empty. - Responseheaders(*flow.Flow) - - // The full HTTP response has been read. - Response(*flow.Flow) -} - -// Base do nothing -type Base struct{} - -func (addon *Base) ClientConnected(*connection.Client) {} -func (addon *Base) ClientDisconnected(*connection.Client) {} -func (addon *Base) ServerConnected(*flow.ConnContext) {} -func (addon *Base) ServerDisconnected(*flow.ConnContext) {} - -func (addon *Base) Requestheaders(*flow.Flow) {} -func (addon *Base) Request(*flow.Flow) {} -func (addon *Base) Responseheaders(*flow.Flow) {} -func (addon *Base) Response(*flow.Flow) {} - -// Log log http record -type Log struct { - Base -} - -func (addon *Log) ClientConnected(client *connection.Client) { - log.Infof("%v client connect\n", client.Conn.RemoteAddr()) -} - -func (addon *Log) ClientDisconnected(client *connection.Client) { - log.Infof("%v client disconnect\n", client.Conn.RemoteAddr()) -} - -func (addon *Log) ServerConnected(connCtx *flow.ConnContext) { - log.Infof("%v server connect %v (%v)\n", connCtx.Client.Conn.RemoteAddr(), connCtx.Server.Address, connCtx.Server.Conn.RemoteAddr()) -} - -func (addon *Log) ServerDisconnected(connCtx *flow.ConnContext) { - log.Infof("%v server disconnect %v (%v)\n", connCtx.Client.Conn.RemoteAddr(), connCtx.Server.Address, connCtx.Server.Conn.RemoteAddr()) -} - -func (addon *Log) Requestheaders(f *flow.Flow) { - log := log.WithField("in", "Log") - start := time.Now() - go func() { - <-f.Done() - var StatusCode int - if f.Response != nil { - StatusCode = f.Response.StatusCode - } - var contentLen int - if f.Response != nil && f.Response.Body != nil { - contentLen = len(f.Response.Body) - } - log.Infof("%v %v %v %v %v - %v ms\n", f.ConnContext.Client.Conn.RemoteAddr(), f.Request.Method, f.Request.URL.String(), StatusCode, contentLen, time.Since(start).Milliseconds()) - }() -} diff --git a/addon/decoder.go b/addon/decoder.go index d7c6a2a..5039391 100644 --- a/addon/decoder.go +++ b/addon/decoder.go @@ -1,13 +1,13 @@ package addon -import "github.com/lqqyt2423/go-mitmproxy/flow" +import "github.com/lqqyt2423/go-mitmproxy/proxy" // decode content-encoding then respond to client type Decoder struct { - Base + proxy.BaseAddon } -func (d *Decoder) Response(f *flow.Flow) { +func (d *Decoder) Response(f *proxy.Flow) { f.Response.ReplaceToDecodedBody() } diff --git a/addon/dumper.go b/addon/dumper.go index e9630c8..1b68ecb 100644 --- a/addon/dumper.go +++ b/addon/dumper.go @@ -9,30 +9,40 @@ import ( "strings" "unicode" - "github.com/lqqyt2423/go-mitmproxy/flow" + "github.com/lqqyt2423/go-mitmproxy/proxy" + log "github.com/sirupsen/logrus" ) type Dumper struct { - Base + proxy.BaseAddon + out io.Writer level int // 0: header 1: header + body - Out io.Writer } -func NewDumper(file string, level int) *Dumper { - out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - panic(err) - } - +func NewDumper(out io.Writer, level int) *Dumper { if level != 0 && level != 1 { level = 0 } + return &Dumper{out: out, level: level} +} + +func NewDumperWithFilename(filename string, level int) *Dumper { + out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + panic(err) + } + return NewDumper(out, level) +} - return &Dumper{Out: out, level: level} +func (d *Dumper) Requestheaders(f *proxy.Flow) { + go func() { + <-f.Done() + d.dump(f) + }() } // call when <-f.Done() -func (d *Dumper) dump(f *flow.Flow) { +func (d *Dumper) dump(f *proxy.Flow) { // 参考 httputil.DumpRequest log := log.WithField("in", "Dumper") @@ -53,7 +63,7 @@ func (d *Dumper) dump(f *flow.Flow) { } buf.WriteString("\r\n") - if d.level == 1 && f.Request.Body != nil && len(f.Request.Body) > 0 && CanPrint(f.Request.Body) { + if d.level == 1 && f.Request.Body != nil && len(f.Request.Body) > 0 && canPrint(f.Request.Body) { buf.Write(f.Request.Body) buf.WriteString("\r\n\r\n") } @@ -77,20 +87,13 @@ func (d *Dumper) dump(f *flow.Flow) { buf.WriteString("\r\n\r\n") - _, err = d.Out.Write(buf.Bytes()) + _, err = d.out.Write(buf.Bytes()) if err != nil { log.Error(err) } } -func (d *Dumper) Requestheaders(f *flow.Flow) { - go func() { - <-f.Done() - d.dump(f) - }() -} - -func CanPrint(content []byte) bool { +func canPrint(content []byte) bool { for _, c := range string(content) { if !unicode.IsPrint(c) && !unicode.IsSpace(c) { return false diff --git a/addon/flowmapper/mapper.go b/addon/flowmapper/mapper.go deleted file mode 100644 index 46146d9..0000000 --- a/addon/flowmapper/mapper.go +++ /dev/null @@ -1,96 +0,0 @@ -package flowmapper - -import ( - "io/ioutil" - "path/filepath" - "regexp" - "strings" - - "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/flow" - _log "github.com/sirupsen/logrus" -) - -var log = _log.WithField("at", "changeflow addon") -var httpsRegexp = regexp.MustCompile(`^https://`) - -type Mapper struct { - addon.Base - reqResMap map[string]*flow.Response -} - -func NewMapper(dirname string) *Mapper { - infos, err := ioutil.ReadDir(dirname) - if err != nil { - panic(err) - } - - filenames := make([]string, 0) - - for _, info := range infos { - if info.IsDir() { - continue - } - if !strings.HasSuffix(info.Name(), ".map.txt") { - continue - } - - filenames = append(filenames, filepath.Join(dirname, info.Name())) - } - - if len(filenames) == 0 { - return &Mapper{ - reqResMap: make(map[string]*flow.Response), - } - } - - ch := make(chan interface{}, len(filenames)) - for _, filename := range filenames { - go func(filename string, ch chan<- interface{}) { - f, err := ParseFlowFromFile(filename) - if err != nil { - log.Error(err) - ch <- err - return - } - ch <- f - }(filename, ch) - } - - reqResMap := make(map[string]*flow.Response) - - for i := 0; i < len(filenames); i++ { - flowOrErr := <-ch - if f, ok := flowOrErr.(*flow.Flow); ok { - key := buildReqKey(f.Request) - log.Infof("add request mapper: %v", key) - reqResMap[key] = f.Response - } - } - - return &Mapper{ - reqResMap: reqResMap, - } -} - -func ParseFlowFromFile(filename string) (*flow.Flow, error) { - p, err := NewParserFromFile(filename) - if err != nil { - return nil, err - } - return p.Parse() -} - -func (c *Mapper) Request(f *flow.Flow) { - key := buildReqKey(f.Request) - if resp, ok := c.reqResMap[key]; ok { - f.Response = resp - } -} - -func buildReqKey(req *flow.Request) string { - url := req.URL.String() - url = httpsRegexp.ReplaceAllString(url, "http://") - key := req.Method + " " + url - return key -} diff --git a/addon/flowmapper/parser.go b/addon/mapper.go similarity index 54% rename from addon/flowmapper/parser.go rename to addon/mapper.go index d8a9939..e38b127 100644 --- a/addon/flowmapper/parser.go +++ b/addon/mapper.go @@ -1,46 +1,131 @@ -package flowmapper +package addon import ( "errors" "io/ioutil" "net/http" "net/url" + "path/filepath" "regexp" "strconv" "strings" - "github.com/lqqyt2423/go-mitmproxy/flow" + "github.com/lqqyt2423/go-mitmproxy/proxy" + log "github.com/sirupsen/logrus" ) -type Parser struct { +var httpsRegexp = regexp.MustCompile(`^https://`) + +type Mapper struct { + proxy.BaseAddon + reqResMap map[string]*proxy.Response +} + +func NewMapper(dirname string) *Mapper { + infos, err := ioutil.ReadDir(dirname) + if err != nil { + panic(err) + } + + filenames := make([]string, 0) + + for _, info := range infos { + if info.IsDir() { + continue + } + if !strings.HasSuffix(info.Name(), ".map.txt") { + continue + } + + filenames = append(filenames, filepath.Join(dirname, info.Name())) + } + + if len(filenames) == 0 { + return &Mapper{ + reqResMap: make(map[string]*proxy.Response), + } + } + + ch := make(chan interface{}, len(filenames)) + for _, filename := range filenames { + go func(filename string, ch chan<- interface{}) { + f, err := parseFlowFromFile(filename) + if err != nil { + log.Error(err) + ch <- err + return + } + ch <- f + }(filename, ch) + } + + reqResMap := make(map[string]*proxy.Response) + + for i := 0; i < len(filenames); i++ { + flowOrErr := <-ch + if f, ok := flowOrErr.(*proxy.Flow); ok { + key := buildReqKey(f.Request) + log.Infof("add request mapper: %v", key) + reqResMap[key] = f.Response + } + } + + return &Mapper{ + reqResMap: reqResMap, + } +} + +func parseFlowFromFile(filename string) (*proxy.Flow, error) { + p, err := newMapperParserFromFile(filename) + if err != nil { + return nil, err + } + return p.parse() +} + +func (c *Mapper) Request(f *proxy.Flow) { + key := buildReqKey(f.Request) + if resp, ok := c.reqResMap[key]; ok { + f.Response = resp + } +} + +func buildReqKey(req *proxy.Request) string { + url := req.URL.String() + url = httpsRegexp.ReplaceAllString(url, "http://") + key := req.Method + " " + url + return key +} + +type mapperParser struct { lines []string url string - request *flow.Request - response *flow.Response + request *proxy.Request + response *proxy.Response } -func NewParserFromFile(filename string) (*Parser, error) { +func newMapperParserFromFile(filename string) (*mapperParser, error) { bytes, err := ioutil.ReadFile(filename) if err != nil { return nil, err } - return NewParserFromString(string(bytes)) + return newMapperParserFromString(string(bytes)) } -func NewParserFromString(content string) (*Parser, error) { +func newMapperParserFromString(content string) (*mapperParser, error) { content = strings.TrimSpace(content) lines := strings.Split(content, "\n") if len(lines) == 0 { return nil, errors.New("no lines") } - return &Parser{ + return &mapperParser{ lines: lines, }, nil } -func (p *Parser) Parse() (*flow.Flow, error) { +func (p *mapperParser) parse() (*proxy.Flow, error) { if err := p.parseRequest(); err != nil { return nil, err } @@ -49,13 +134,13 @@ func (p *Parser) Parse() (*flow.Flow, error) { return nil, err } - return &flow.Flow{ + return &proxy.Flow{ Request: p.request, Response: p.response, }, nil } -func (p *Parser) parseRequest() error { +func (p *mapperParser) parseRequest() error { if err := p.parseReqHead(); err != nil { return err } @@ -85,7 +170,7 @@ func (p *Parser) parseRequest() error { return nil } -func (p *Parser) parseReqHead() error { +func (p *mapperParser) parseReqHead() error { line, _ := p.getLine() re := regexp.MustCompile(`^(GET|POST|PUT|DELETE)\s+?(.+)`) matches := re.FindStringSubmatch(line) @@ -93,7 +178,7 @@ func (p *Parser) parseReqHead() error { return errors.New("request head parse error") } - p.request = &flow.Request{ + p.request = &proxy.Request{ Method: matches[1], } p.url = matches[2] @@ -101,7 +186,7 @@ func (p *Parser) parseReqHead() error { return nil } -func (p *Parser) parseHeader() (http.Header, error) { +func (p *mapperParser) parseHeader() (http.Header, error) { header := make(http.Header) re := regexp.MustCompile(`^([\w-]+?):\s*(.+)$`) @@ -127,7 +212,7 @@ func (p *Parser) parseHeader() (http.Header, error) { return header, nil } -func (p *Parser) parseReqBody() { +func (p *mapperParser) parseReqBody() { bodyLines := make([]string, 0) for { @@ -155,7 +240,7 @@ func (p *Parser) parseReqBody() { p.request.Body = []byte(body) } -func (p *Parser) parseResponse() error { +func (p *mapperParser) parseResponse() error { if err := p.parseResHead(); err != nil { return err } @@ -175,7 +260,7 @@ func (p *Parser) parseResponse() error { return nil } -func (p *Parser) parseResHead() error { +func (p *mapperParser) parseResHead() error { line, ok := p.getLine() if !ok { return errors.New("response no head line") @@ -188,14 +273,14 @@ func (p *Parser) parseResHead() error { } code, _ := strconv.Atoi(matches[1]) - p.response = &flow.Response{ + p.response = &proxy.Response{ StatusCode: code, } return nil } -func (p *Parser) getLine() (string, bool) { +func (p *mapperParser) getLine() (string, bool) { if len(p.lines) == 0 { return "", false } diff --git a/addon/flowmapper/parser_test.go b/addon/mapper_test.go similarity index 88% rename from addon/flowmapper/parser_test.go rename to addon/mapper_test.go index 623971f..8b9c5ea 100644 --- a/addon/flowmapper/parser_test.go +++ b/addon/mapper_test.go @@ -1,4 +1,4 @@ -package flowmapper +package addon import "testing" @@ -14,11 +14,11 @@ HTTP/1.1 200 ok ` - p, err := NewParserFromString(content) + p, err := newMapperParserFromString(content) if err != nil { t.Fatal(err) } - f, err := p.Parse() + f, err := p.parse() if err != nil { t.Fatal(err) } diff --git a/cmd/go-mitmproxy/main.go b/cmd/go-mitmproxy/main.go index cfe6eee..233fe65 100644 --- a/cmd/go-mitmproxy/main.go +++ b/cmd/go-mitmproxy/main.go @@ -7,9 +7,8 @@ import ( "os" "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/addon/flowmapper" - "github.com/lqqyt2423/go-mitmproxy/addon/web" "github.com/lqqyt2423/go-mitmproxy/proxy" + "github.com/lqqyt2423/go-mitmproxy/web" log "github.com/sirupsen/logrus" ) @@ -82,16 +81,16 @@ func main() { log.Infof("go-mitmproxy version %v\n", p.Version) - p.AddAddon(&addon.Log{}) + p.AddAddon(&proxy.LogAddon{}) p.AddAddon(web.NewWebAddon(config.webAddr)) if config.dump != "" { - dumper := addon.NewDumper(config.dump, config.dumpLevel) + dumper := addon.NewDumperWithFilename(config.dump, config.dumpLevel) p.AddAddon(dumper) } if config.mapperDir != "" { - mapper := flowmapper.NewMapper(config.mapperDir) + mapper := addon.NewMapper(config.mapperDir) p.AddAddon(mapper) } diff --git a/connection/connection.go b/connection/connection.go deleted file mode 100644 index aec42f1..0000000 --- a/connection/connection.go +++ /dev/null @@ -1,35 +0,0 @@ -package connection - -import ( - "net" - "net/http" - - uuid "github.com/satori/go.uuid" -) - -type Client struct { - Id uuid.UUID - Conn net.Conn - Tls bool -} - -func NewClient(c net.Conn) *Client { - return &Client{ - Id: uuid.NewV4(), - Conn: c, - Tls: false, - } -} - -type Server struct { - Id uuid.UUID - Conn net.Conn - Client *http.Client - Address string -} - -func NewServer() *Server { - return &Server{ - Id: uuid.NewV4(), - } -} diff --git a/examples/change-html/main.go b/examples/change-html/main.go index 6da61e1..bfb86c0 100644 --- a/examples/change-html/main.go +++ b/examples/change-html/main.go @@ -5,20 +5,17 @@ import ( "strconv" "strings" - log "github.com/sirupsen/logrus" - - "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy" + log "github.com/sirupsen/logrus" ) var titleRegexp = regexp.MustCompile("()(.*?)()") type ChangeHtml struct { - addon.Base + proxy.BaseAddon } -func (c *ChangeHtml) Response(f *flow.Flow) { +func (c *ChangeHtml) Response(f *proxy.Flow) { contentType := f.Response.Header.Get("Content-Type") if !strings.Contains(contentType, "text/html") { return diff --git a/examples/http-add-header/main.go b/examples/http-add-header/main.go index 79f8b26..dcd9ea2 100644 --- a/examples/http-add-header/main.go +++ b/examples/http-add-header/main.go @@ -5,17 +5,15 @@ import ( log "github.com/sirupsen/logrus" - "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy" ) type AddHeader struct { - addon.Base + proxy.BaseAddon count int } -func (a *AddHeader) Responseheaders(f *flow.Flow) { +func (a *AddHeader) Responseheaders(f *proxy.Flow) { a.count += 1 f.Response.Header.Add("x-count", strconv.Itoa(a.count)) } diff --git a/flow/conncontext.go b/flow/conncontext.go deleted file mode 100644 index 7d7565d..0000000 --- a/flow/conncontext.go +++ /dev/null @@ -1,119 +0,0 @@ -package flow - -import ( - "context" - "crypto/tls" - "net" - "net/http" - - "github.com/lqqyt2423/go-mitmproxy/connection" -) - -var ConnContextKey = new(struct{}) - -type ConnContext struct { - Client *connection.Client - Server *connection.Server -} - -func NewConnContext(c net.Conn) *ConnContext { - client := connection.NewClient(c) - return &ConnContext{ - Client: client, - } -} - -func (connCtx *ConnContext) InitHttpServer(sslInsecure bool, connWrap func(net.Conn) net.Conn, whenConnected func()) { - if connCtx.Server != nil { - return - } - if connCtx.Client.Tls { - return - } - - server := connection.NewServer() - server.Client = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - - // todo: change here - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - c, err := (&net.Dialer{ - // Timeout: 30 * time.Second, - // KeepAlive: 30 * time.Second, - }).DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - - cw := connWrap(c) - server.Conn = cw - server.Address = addr - defer whenConnected() - return cw, nil - }, - ForceAttemptHTTP2: false, // disable http2 - - DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true. - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: sslInsecure, - KeyLogWriter: GetTlsKeyLogWriter(), - }, - }, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - // 禁止自动重定向 - return http.ErrUseLastResponse - }, - } - connCtx.Server = server -} - -func (connCtx *ConnContext) InitHttpsServer(sslInsecure bool, connWrap func(net.Conn) net.Conn, whenConnected func()) { - if connCtx.Server != nil { - return - } - if !connCtx.Client.Tls { - return - } - - server := connection.NewServer() - server.Client = &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - - DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - log.Debugln("in https DialTLSContext") - - plainConn, err := (&net.Dialer{}).DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - - cw := connWrap(plainConn) - server.Conn = cw - server.Address = addr - whenConnected() - - firstTLSHost, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - cfg := &tls.Config{ - InsecureSkipVerify: sslInsecure, - KeyLogWriter: GetTlsKeyLogWriter(), - ServerName: firstTLSHost, - } - tlsConn := tls.Client(cw, cfg) - return tlsConn, nil - }, - ForceAttemptHTTP2: false, // disable http2 - - DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true. - }, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - // 禁止自动重定向 - return http.ErrUseLastResponse - }, - } - connCtx.Server = server -} diff --git a/flow/helper.go b/flow/helper.go deleted file mode 100644 index 6dee72d..0000000 --- a/flow/helper.go +++ /dev/null @@ -1,29 +0,0 @@ -package flow - -import ( - "io" - "os" - "sync" -) - -// Wireshark 解析 https 设置 -var tlsKeyLogWriter io.Writer -var tlsKeyLogOnce sync.Once - -func GetTlsKeyLogWriter() io.Writer { - tlsKeyLogOnce.Do(func() { - logfile := os.Getenv("SSLKEYLOGFILE") - if logfile == "" { - return - } - - writer, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - log.WithField("in", "GetTlsKeyLogWriter").Debug(err) - return - } - - tlsKeyLogWriter = writer - }) - return tlsKeyLogWriter -} diff --git a/proxy/addon.go b/proxy/addon.go new file mode 100644 index 0000000..56a18f7 --- /dev/null +++ b/proxy/addon.go @@ -0,0 +1,81 @@ +package proxy + +import ( + "time" +) + +type Addon interface { + // A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests. + ClientConnected(*ClientConn) + + // A client connection has been closed (either by us or the client). + ClientDisconnected(*ClientConn) + + // Mitmproxy has connected to a server. + ServerConnected(*ConnContext) + + // A server connection has been closed (either by us or the server). + ServerDisconnected(*ConnContext) + + // HTTP request headers were successfully read. At this point, the body is empty. + Requestheaders(*Flow) + + // The full HTTP request has been read. + Request(*Flow) + + // HTTP response headers were successfully read. At this point, the body is empty. + Responseheaders(*Flow) + + // The full HTTP response has been read. + Response(*Flow) +} + +// BaseAddon do nothing +type BaseAddon struct{} + +func (addon *BaseAddon) ClientConnected(*ClientConn) {} +func (addon *BaseAddon) ClientDisconnected(*ClientConn) {} +func (addon *BaseAddon) ServerConnected(*ConnContext) {} +func (addon *BaseAddon) ServerDisconnected(*ConnContext) {} + +func (addon *BaseAddon) Requestheaders(*Flow) {} +func (addon *BaseAddon) Request(*Flow) {} +func (addon *BaseAddon) Responseheaders(*Flow) {} +func (addon *BaseAddon) Response(*Flow) {} + +// LogAddon log connection and flow +type LogAddon struct { + BaseAddon +} + +func (addon *LogAddon) ClientConnected(client *ClientConn) { + log.Infof("%v client connect\n", client.Conn.RemoteAddr()) +} + +func (addon *LogAddon) ClientDisconnected(client *ClientConn) { + log.Infof("%v client disconnect\n", client.Conn.RemoteAddr()) +} + +func (addon *LogAddon) ServerConnected(connCtx *ConnContext) { + log.Infof("%v server connect %v (%v)\n", connCtx.ClientConn.Conn.RemoteAddr(), connCtx.ServerConn.Address, connCtx.ServerConn.Conn.RemoteAddr()) +} + +func (addon *LogAddon) ServerDisconnected(connCtx *ConnContext) { + log.Infof("%v server disconnect %v (%v)\n", connCtx.ClientConn.Conn.RemoteAddr(), connCtx.ServerConn.Address, connCtx.ServerConn.Conn.RemoteAddr()) +} + +func (addon *LogAddon) Requestheaders(f *Flow) { + start := time.Now() + go func() { + <-f.Done() + var StatusCode int + if f.Response != nil { + StatusCode = f.Response.StatusCode + } + var contentLen int + if f.Response != nil && f.Response.Body != nil { + contentLen = len(f.Response.Body) + } + log.Infof("%v %v %v %v %v - %v ms\n", f.ConnContext.ClientConn.Conn.RemoteAddr(), f.Request.Method, f.Request.URL.String(), StatusCode, contentLen, time.Since(start).Milliseconds()) + }() +} diff --git a/proxy/connection.go b/proxy/connection.go new file mode 100644 index 0000000..0cd606e --- /dev/null +++ b/proxy/connection.go @@ -0,0 +1,232 @@ +package proxy + +import ( + "context" + "crypto/tls" + "net" + "net/http" + + uuid "github.com/satori/go.uuid" +) + +// client connection +type ClientConn struct { + Id uuid.UUID + Conn net.Conn + Tls bool +} + +func newClientConn(c net.Conn) *ClientConn { + return &ClientConn{ + Id: uuid.NewV4(), + Conn: c, + Tls: false, + } +} + +// server connection +type ServerConn struct { + Id uuid.UUID + Address string + Conn net.Conn + + client *http.Client +} + +func newServerConn() *ServerConn { + return &ServerConn{ + Id: uuid.NewV4(), + } +} + +// connection context ctx key +var connContextKey = new(struct{}) + +// connection context +type ConnContext struct { + ClientConn *ClientConn + ServerConn *ServerConn + + proxy *Proxy +} + +func newConnContext(c net.Conn, proxy *Proxy) *ConnContext { + clientConn := newClientConn(c) + return &ConnContext{ + ClientConn: clientConn, + proxy: proxy, + } +} + +func (connCtx *ConnContext) InitHttpServerConn() { + if connCtx.ServerConn != nil { + return + } + if connCtx.ClientConn.Tls { + return + } + + serverConn := newServerConn() + serverConn.client = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := (&net.Dialer{}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + cw := &wrapServerConn{ + Conn: c, + proxy: connCtx.proxy, + connCtx: connCtx, + } + serverConn.Conn = cw + serverConn.Address = addr + defer func() { + for _, addon := range connCtx.proxy.Addons { + addon.ServerConnected(connCtx) + } + }() + return cw, nil + }, + ForceAttemptHTTP2: false, // disable http2 + DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true. + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: connCtx.proxy.Opts.SslInsecure, + KeyLogWriter: getTlsKeyLogWriter(), + }, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // 禁止自动重定向 + return http.ErrUseLastResponse + }, + } + connCtx.ServerConn = serverConn +} + +func (connCtx *ConnContext) InitHttpsServerConn() { + if connCtx.ServerConn != nil { + return + } + if !connCtx.ClientConn.Tls { + return + } + + ServerConn := newServerConn() + ServerConn.client = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + log.Debugln("in https DialTLSContext") + plainConn, err := (&net.Dialer{}).DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + cw := &wrapServerConn{ + Conn: plainConn, + proxy: connCtx.proxy, + connCtx: connCtx, + } + ServerConn.Conn = cw + ServerConn.Address = addr + + for _, addon := range connCtx.proxy.Addons { + addon.ServerConnected(connCtx) + } + + firstTLSHost, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + cfg := &tls.Config{ + InsecureSkipVerify: connCtx.proxy.Opts.SslInsecure, + KeyLogWriter: getTlsKeyLogWriter(), + ServerName: firstTLSHost, + } + tlsConn := tls.Client(cw, cfg) + return tlsConn, nil + }, + ForceAttemptHTTP2: false, // disable http2 + DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true. + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + // 禁止自动重定向 + return http.ErrUseLastResponse + }, + } + connCtx.ServerConn = ServerConn +} + +// wrap tcpConn for remote client +type wrapClientConn struct { + net.Conn + proxy *Proxy + connCtx *ConnContext + closed bool + closeErr error +} + +func (c *wrapClientConn) Close() error { + log.Debugln("in wrapClientConn close") + if c.closed { + return c.closeErr + } + + c.closed = true + c.closeErr = c.Conn.Close() + + for _, addon := range c.proxy.Addons { + addon.ClientDisconnected(c.connCtx.ClientConn) + } + + if c.connCtx.ServerConn != nil && c.connCtx.ServerConn.Conn != nil { + c.connCtx.ServerConn.Conn.Close() + } + + return c.closeErr +} + +// wrap tcpListener for remote client +type wrapListener struct { + net.Listener + proxy *Proxy +} + +func (l *wrapListener) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + return &wrapClientConn{ + Conn: c, + proxy: l.proxy, + }, nil +} + +// wrap tcpConn for remote server +type wrapServerConn struct { + net.Conn + proxy *Proxy + connCtx *ConnContext + closed bool + closeErr error +} + +func (c *wrapServerConn) Close() error { + log.Debugln("in wrapServerConn close") + if c.closed { + return c.closeErr + } + + c.closed = true + c.closeErr = c.Conn.Close() + + for _, addon := range c.proxy.Addons { + addon.ServerDisconnected(c.connCtx) + } + + c.connCtx.ClientConn.Conn.Close() + + return c.closeErr +} diff --git a/flow/flow.go b/proxy/flow.go similarity index 91% rename from flow/flow.go rename to proxy/flow.go index 52abc42..2bec3b6 100644 --- a/flow/flow.go +++ b/proxy/flow.go @@ -1,4 +1,4 @@ -package flow +package proxy import ( "encoding/json" @@ -7,11 +7,9 @@ import ( "net/url" uuid "github.com/satori/go.uuid" - _log "github.com/sirupsen/logrus" ) -var log = _log.WithField("at", "flow") - +// flow http request type Request struct { Method string URL *url.URL @@ -22,6 +20,20 @@ type Request struct { raw *http.Request } +func newRequest(req *http.Request) *Request { + return &Request{ + Method: req.Method, + URL: req.URL, + Proto: req.Proto, + Header: req.Header, + raw: req, + } +} + +func (r *Request) Raw() *http.Request { + return r.raw +} + func (req *Request) MarshalJSON() ([]byte, error) { r := make(map[string]interface{}) r["method"] = req.Method @@ -79,20 +91,7 @@ func (req *Request) UnmarshalJSON(data []byte) error { return nil } -func NewRequest(req *http.Request) *Request { - return &Request{ - Method: req.Method, - URL: req.URL, - Proto: req.Proto, - Header: req.Header, - raw: req, - } -} - -func (r *Request) Raw() *http.Request { - return r.raw -} - +// flow http response type Response struct { StatusCode int `json:"statusCode"` Header http.Header `json:"header"` @@ -103,31 +102,24 @@ type Response struct { decodedErr error } +// flow type Flow struct { - *Request - *Response + Id uuid.UUID + ConnContext *ConnContext + Request *Request + Response *Response // https://docs.mitmproxy.org/stable/overview-features/#streaming // 如果为 true，则不缓冲 Request.Body 和 Response.Body，且不进入之后的 Addon.Request 和 Addon.Response Stream bool - done chan struct{} - - Id uuid.UUID - ConnContext *ConnContext -} -func (f *Flow) MarshalJSON() ([]byte, error) { - j := make(map[string]interface{}) - j["id"] = f.Id - j["request"] = f.Request - j["response"] = f.Response - return json.Marshal(j) + done chan struct{} } -func NewFlow() *Flow { +func newFlow() *Flow { return &Flow{ - done: make(chan struct{}), Id: uuid.NewV4(), + done: make(chan struct{}), } } @@ -135,6 +127,14 @@ func (f *Flow) Done() <-chan struct{} { return f.done } -func (f *Flow) Finish() { +func (f *Flow) finish() { close(f.done) } + +func (f *Flow) MarshalJSON() ([]byte, error) { + j := make(map[string]interface{}) + j["id"] = f.Id + j["request"] = f.Request + j["response"] = f.Response + return json.Marshal(j) +} diff --git a/flow/encoding.go b/proxy/flowencoding.go similarity index 89% rename from flow/encoding.go rename to proxy/flowencoding.go index f60dc09..8cf074f 100644 --- a/flow/encoding.go +++ b/proxy/flowencoding.go @@ -1,4 +1,4 @@ -package flow +package proxy import ( "bytes" @@ -12,9 +12,7 @@ import ( "github.com/andybalholm/brotli" ) -// handle http header: content-encoding - -var EncodingNotSupport = errors.New("content-encoding not support") +var errEncodingNotSupport = errors.New("content-encoding not support") var textContentTypes = []string{ "text", @@ -59,7 +57,7 @@ func (r *Response) DecodedBody() ([]byte, error) { return r.decodedBody, nil } - decodedBody, decodedErr := Decode(enc, r.Body) + decodedBody, decodedErr := decode(enc, r.Body) if decodedErr != nil { r.decodedErr = decodedErr log.Error(r.decodedErr) @@ -83,7 +81,7 @@ func (r *Response) ReplaceToDecodedBody() { r.Header.Del("Transfer-Encoding") } -func Decode(enc string, body []byte) ([]byte, error) { +func decode(enc string, body []byte) ([]byte, error) { if enc == "gzip" { dreader, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { @@ -117,5 +115,5 @@ func Decode(enc string, body []byte) ([]byte, error) { return buf.Bytes(), nil } - return nil, EncodingNotSupport + return nil, errEncodingNotSupport } diff --git a/proxy/helper.go b/proxy/helper.go index 142dabb..5ef6600 100644 --- a/proxy/helper.go +++ b/proxy/helper.go @@ -3,12 +3,14 @@ package proxy import ( "bytes" "io" + "os" "strings" + "sync" _log "github.com/sirupsen/logrus" ) -var NormalErrMsgs []string = []string{ +var normalErrMsgs []string = []string{ "read: connection reset by peer", "write: broken pipe", "i/o timeout", @@ -20,10 +22,10 @@ var NormalErrMsgs []string = []string{ } // 仅打印预料之外的错误信息 -func LogErr(log *_log.Entry, err error) (loged bool) { +func logErr(log *_log.Entry, err error) (loged bool) { msg := err.Error() - for _, str := range NormalErrMsgs { + for _, str := range normalErrMsgs { if strings.Contains(msg, str) { log.Debug(err) return @@ -61,7 +63,7 @@ func transfer(log *_log.Entry, a, b io.ReadWriteCloser) { for i := 0; i < 2; i++ { if err := <-errChan; err != nil { - LogErr(log, err) + logErr(log, err) return // 如果有错误，直接返回 } } @@ -70,7 +72,7 @@ func transfer(log *_log.Entry, a, b io.ReadWriteCloser) { // 尝试将 Reader 读取至 buffer 中 // 如果未达到 limit，则成功读取进入 buffer // 否则 buffer 返回 nil，且返回新 Reader，状态为未读取前 -func ReaderToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) { +func readerToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) { buf := bytes.NewBuffer(make([]byte, 0)) lr := io.LimitReader(r, limit) @@ -88,3 +90,25 @@ func ReaderToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) { // 返回 buffer return buf.Bytes(), nil, nil } + +// Wireshark 解析 https 设置 +var tlsKeyLogWriter io.Writer +var tlsKeyLogOnce sync.Once + +func getTlsKeyLogWriter() io.Writer { + tlsKeyLogOnce.Do(func() { + logfile := os.Getenv("SSLKEYLOGFILE") + if logfile == "" { + return + } + + writer, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.WithField("in", "getTlsKeyLogWriter").Debug(err) + return + } + + tlsKeyLogWriter = writer + }) + return tlsKeyLogWriter +} diff --git a/proxy/interceptor.go b/proxy/interceptor.go index 7fef80d..620ccc0 100644 --- a/proxy/interceptor.go +++ b/proxy/interceptor.go @@ -6,7 +6,7 @@ import ( ) // 拦截 https 流量通用接口 -type Interceptor interface { +type interceptor interface { // 初始化 Start() error // 传入当前客户端 req @@ -14,12 +14,12 @@ type Interceptor interface { } // 直接转发 https 流量 -type Forward struct{} +type forward struct{} -func (i *Forward) Start() error { +func (i *forward) Start() error { return nil } -func (i *Forward) Dial(req *http.Request) (net.Conn, error) { +func (i *forward) Dial(req *http.Request) (net.Conn, error) { return net.Dial("tcp", req.Host) } diff --git a/proxy/middle.go b/proxy/middle.go index c2a30d0..1f9ac4a 100644 --- a/proxy/middle.go +++ b/proxy/middle.go @@ -9,20 +9,10 @@ import ( "strings" "github.com/lqqyt2423/go-mitmproxy/cert" - "github.com/lqqyt2423/go-mitmproxy/flow" ) // 模拟了标准库中 server 运行，目的是仅通过当前进程内存转发 socket 数据，不需要经过 tcp 或 unix socket -// mock net.Listener -type middleListener struct { - connChan chan net.Conn -} - -func (l *middleListener) Accept() (net.Conn, error) { return <-l.connChan, nil } -func (l *middleListener) Close() error { return nil } -func (l *middleListener) Addr() net.Addr { return nil } - type pipeAddr struct { remoteAddr string } @@ -30,20 +20,13 @@ type pipeAddr struct { func (pipeAddr) Network() string { return "pipe" } func (a *pipeAddr) String() string { return a.remoteAddr } -// 建立客户端和服务端通信的通道 -func newPipes(req *http.Request) (net.Conn, *pipeConn) { - client, srv := net.Pipe() - server := newPipeConn(srv, req) - return client, server -} - // add Peek method for conn type pipeConn struct { net.Conn r *bufio.Reader host string remoteAddr string - connContext *flow.ConnContext + connContext *ConnContext } func newPipeConn(c net.Conn, req *http.Request) *pipeConn { @@ -52,7 +35,7 @@ func newPipeConn(c net.Conn, req *http.Request) *pipeConn { r: bufio.NewReader(c), host: req.Host, remoteAddr: req.RemoteAddr, - connContext: req.Context().Value(flow.ConnContextKey).(*flow.ConnContext), + connContext: req.Context().Value(connContextKey).(*ConnContext), } } @@ -68,33 +51,49 @@ func (c *pipeConn) RemoteAddr() net.Addr { return &pipeAddr{remoteAddr: c.remoteAddr} } -// Middle: man-in-the-middle -type Middle struct { - Proxy *Proxy - CA *cert.CA - Listener net.Listener - Server *http.Server +// 建立客户端和服务端通信的通道 +func newPipes(req *http.Request) (net.Conn, *pipeConn) { + client, srv := net.Pipe() + server := newPipeConn(srv, req) + return client, server +} + +// mock net.Listener +type middleListener struct { + connChan chan net.Conn } -func NewMiddle(proxy *Proxy, caPath string) (Interceptor, error) { - ca, err := cert.NewCA(caPath) +func (l *middleListener) Accept() (net.Conn, error) { return <-l.connChan, nil } +func (l *middleListener) Close() error { return nil } +func (l *middleListener) Addr() net.Addr { return nil } + +// middle: man-in-the-middle server +type middle struct { + proxy *Proxy + ca *cert.CA + listener *middleListener + server *http.Server +} + +func newMiddle(proxy *Proxy) (interceptor, error) { + ca, err := cert.NewCA(proxy.Opts.CaRootPath) if err != nil { return nil, err } - m := &Middle{ - Proxy: proxy, - CA: ca, + m := &middle{ + proxy: proxy, + ca: ca, + listener: &middleListener{ + connChan: make(chan net.Conn), + }, } server := &http.Server{ Handler: m, - // IdleTimeout: 5 * time.Second, - ConnContext: func(ctx context.Context, c net.Conn) context.Context { - return context.WithValue(ctx, flow.ConnContextKey, c.(*tls.Conn).NetConn().(*pipeConn).connContext) + return context.WithValue(ctx, connContextKey, c.(*tls.Conn).NetConn().(*pipeConn).connContext) }, - TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), // disable http2 TLSConfig: &tls.Config{ GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) { @@ -103,28 +102,25 @@ func NewMiddle(proxy *Proxy, caPath string) (Interceptor, error) { }, }, } - - m.Server = server - m.Listener = &middleListener{make(chan net.Conn)} - + m.server = server return m, nil } -func (m *Middle) Start() error { - return m.Server.ServeTLS(m.Listener, "", "") +func (m *middle) Start() error { + return m.server.ServeTLS(m.listener, "", "") } // todo: should block until ServerConnected -func (m *Middle) Dial(req *http.Request) (net.Conn, error) { +func (m *middle) Dial(req *http.Request) (net.Conn, error) { pipeClientConn, pipeServerConn := newPipes(req) go m.intercept(pipeServerConn) return pipeClientConn, nil } -func (m *Middle) ServeHTTP(res http.ResponseWriter, req *http.Request) { +func (m *middle) ServeHTTP(res http.ResponseWriter, req *http.Request) { if strings.EqualFold(req.Header.Get("Connection"), "Upgrade") && strings.EqualFold(req.Header.Get("Upgrade"), "websocket") { // wss - DefaultWebSocket.WSS(res, req) + defaultWebSocket.wss(res, req) return } @@ -134,14 +130,14 @@ func (m *Middle) ServeHTTP(res http.ResponseWriter, req *http.Request) { if req.URL.Host == "" { req.URL.Host = req.Host } - m.Proxy.ServeHTTP(res, req) + m.proxy.ServeHTTP(res, req) } // 解析 connect 流量 // 如果是 tls 流量，则进入 listener.Accept => Middle.ServeHTTP // 否则很可能是 ws 流量 -func (m *Middle) intercept(pipeServerConn *pipeConn) { - log := log.WithField("in", "Middle.intercept").WithField("host", pipeServerConn.host) +func (m *middle) intercept(pipeServerConn *pipeConn) { + log := log.WithField("in", "middle.intercept").WithField("host", pipeServerConn.host) buf, err := pipeServerConn.Peek(3) if err != nil { @@ -153,25 +149,11 @@ func (m *Middle) intercept(pipeServerConn *pipeConn) { // https://github.com/mitmproxy/mitmproxy/blob/main/mitmproxy/net/tls.py is_tls_record_magic if buf[0] == 0x16 && buf[1] == 0x03 && buf[2] <= 0x03 { // tls - pipeServerConn.connContext.Client.Tls = true - pipeServerConn.connContext.InitHttpsServer( - m.Proxy.Opts.SslInsecure, - func(c net.Conn) net.Conn { - return &serverConn{ - Conn: c, - proxy: m.Proxy, - connCtx: pipeServerConn.connContext, - } - }, - func() { - for _, addon := range m.Proxy.Addons { - addon.ServerConnected(pipeServerConn.connContext) - } - }, - ) - m.Listener.(*middleListener).connChan <- pipeServerConn + pipeServerConn.connContext.ClientConn.Tls = true + pipeServerConn.connContext.InitHttpsServerConn() + m.listener.connChan <- pipeServerConn } else { // ws - DefaultWebSocket.WS(pipeServerConn, pipeServerConn.host) + defaultWebSocket.ws(pipeServerConn, pipeServerConn.host) } } diff --git a/proxy/proxy.go b/proxy/proxy.go index d1d9f2a..b1fa4d5 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -7,8 +7,6 @@ import ( "net" "net/http" - "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/flow" _log "github.com/sirupsen/logrus" ) @@ -23,120 +21,48 @@ type Options struct { } type Proxy struct { - Opts *Options - Version string - Server *http.Server - Interceptor Interceptor - Addons []addon.Addon -} + Opts *Options + Version string + Addons []Addon -type proxyListener struct { - net.Listener - proxy *Proxy + server *http.Server + interceptor interceptor } -func (l *proxyListener) Accept() (net.Conn, error) { - c, err := l.Listener.Accept() - if err != nil { - return nil, err - } - - return &proxyConn{ - Conn: c, - proxy: l.proxy, - }, nil -} - -type proxyConn struct { - net.Conn - proxy *Proxy - connCtx *flow.ConnContext - closed bool - closeErr error -} - -func (c *proxyConn) Close() error { - log.Debugln("in proxyConn close") - if c.closed { - return c.closeErr - } - - c.closed = true - c.closeErr = c.Conn.Close() - - for _, addon := range c.proxy.Addons { - addon.ClientDisconnected(c.connCtx.Client) - } - - if c.connCtx.Server != nil && c.connCtx.Server.Conn != nil { - c.connCtx.Server.Conn.Close() - } - - return c.closeErr -} - -type serverConn struct { - net.Conn - proxy *Proxy - connCtx *flow.ConnContext - closed bool - closeErr error -} - -func (c *serverConn) Close() error { - log.Debugln("in http serverConn close") - if c.closed { - return c.closeErr +func NewProxy(opts *Options) (*Proxy, error) { + if opts.StreamLargeBodies <= 0 { + opts.StreamLargeBodies = 1024 * 1024 * 5 // default: 5mb } - c.closed = true - c.closeErr = c.Conn.Close() - - for _, addon := range c.proxy.Addons { - addon.ServerDisconnected(c.connCtx) + proxy := &Proxy{ + Opts: opts, + Version: "1.0.0", + Addons: make([]Addon, 0), } - c.connCtx.Client.Conn.Close() - - return c.closeErr -} - -func NewProxy(opts *Options) (*Proxy, error) { - proxy := new(Proxy) - proxy.Opts = opts - proxy.Version = "0.2.0" - - proxy.Server = &http.Server{ + proxy.server = &http.Server{ Addr: opts.Addr, Handler: proxy, - // IdleTimeout: 5 * time.Second, - ConnContext: func(ctx context.Context, c net.Conn) context.Context { - connCtx := flow.NewConnContext(c) + connCtx := newConnContext(c, proxy) for _, addon := range proxy.Addons { - addon.ClientConnected(connCtx.Client) + addon.ClientConnected(connCtx.ClientConn) } - c.(*proxyConn).connCtx = connCtx - return context.WithValue(ctx, flow.ConnContextKey, connCtx) + c.(*wrapClientConn).connCtx = connCtx + return context.WithValue(ctx, connContextKey, connCtx) }, } - interceptor, err := NewMiddle(proxy, opts.CaRootPath) + interceptor, err := newMiddle(proxy) if err != nil { return nil, err } - proxy.Interceptor = interceptor - - if opts.StreamLargeBodies <= 0 { - opts.StreamLargeBodies = 1024 * 1024 * 5 // default: 5mb - } - - proxy.Addons = make([]addon.Addon, 0) + proxy.interceptor = interceptor return proxy, nil } -func (proxy *Proxy) AddAddon(addon addon.Addon) { +func (proxy *Proxy) AddAddon(addon Addon) { proxy.Addons = append(proxy.Addons, addon) } @@ -144,8 +70,8 @@ func (proxy *Proxy) Start() error { errChan := make(chan error) go func() { - log.Infof("Proxy start listen at %v\n", proxy.Server.Addr) - addr := proxy.Server.Addr + log.Infof("Proxy start listen at %v\n", proxy.server.Addr) + addr := proxy.server.Addr if addr == "" { addr = ":http" } @@ -154,16 +80,16 @@ func (proxy *Proxy) Start() error { errChan <- err return } - pln := &proxyListener{ + pln := &wrapListener{ Listener: ln, proxy: proxy, } - err = proxy.Server.Serve(pln) + err = proxy.server.Serve(pln) errChan <- err }() go func() { - err := proxy.Interceptor.Start() + err := proxy.interceptor.Start() errChan <- err }() @@ -194,7 +120,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { return } - reply := func(response *flow.Response, body io.Reader) { + reply := func(response *Response, body io.Reader) { if response.Header != nil { for key, value := range response.Header { for _, v := range value { @@ -207,12 +133,12 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { if body != nil { _, err := io.Copy(res, body) if err != nil { - LogErr(log, err) + logErr(log, err) } } else if response.Body != nil && len(response.Body) > 0 { _, err := res.Write(response.Body) if err != nil { - LogErr(log, err) + logErr(log, err) } } } @@ -224,10 +150,10 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { } }() - f := flow.NewFlow() - f.Request = flow.NewRequest(req) - f.ConnContext = req.Context().Value(flow.ConnContextKey).(*flow.ConnContext) - defer f.Finish() + f := newFlow() + f.Request = newRequest(req) + f.ConnContext = req.Context().Value(connContextKey).(*ConnContext) + defer f.finish() // trigger addon event Requestheaders for _, addon := range proxy.Addons { @@ -241,7 +167,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { // Read request body var reqBody io.Reader = req.Body if !f.Stream { - reqBuf, r, err := ReaderToBuffer(req.Body, proxy.Opts.StreamLargeBodies) + reqBuf, r, err := readerToBuffer(req.Body, proxy.Opts.StreamLargeBodies) reqBody = r if err != nil { log.Error(err) @@ -280,31 +206,16 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { } } - f.ConnContext.InitHttpServer( - proxy.Opts.SslInsecure, - func(c net.Conn) net.Conn { - return &serverConn{ - Conn: c, - proxy: proxy, - connCtx: f.ConnContext, - } - }, - func() { - for _, addon := range proxy.Addons { - addon.ServerConnected(f.ConnContext) - } - }, - ) - - proxyRes, err := f.ConnContext.Server.Client.Do(proxyReq) + f.ConnContext.InitHttpServerConn() + proxyRes, err := f.ConnContext.ServerConn.client.Do(proxyReq) if err != nil { - LogErr(log, err) + logErr(log, err) res.WriteHeader(502) return } defer proxyRes.Body.Close() - f.Response = &flow.Response{ + f.Response = &Response{ StatusCode: proxyRes.StatusCode, Header: proxyRes.Header, } @@ -321,7 +232,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) { // Read response body var resBody io.Reader = proxyRes.Body if !f.Stream { - resBuf, r, err := ReaderToBuffer(proxyRes.Body, proxy.Opts.StreamLargeBodies) + resBuf, r, err := readerToBuffer(proxyRes.Body, proxy.Opts.StreamLargeBodies) resBody = r if err != nil { log.Error(err) @@ -352,7 +263,7 @@ func (proxy *Proxy) handleConnect(res http.ResponseWriter, req *http.Request) { log.Debug("receive connect") - conn, err := proxy.Interceptor.Dial(req) + conn, err := proxy.interceptor.Dial(req) if err != nil { log.Error(err) res.WriteHeader(502) diff --git a/proxy/websocket.go b/proxy/websocket.go index f2a50ff..de1b07d 100644 --- a/proxy/websocket.go +++ b/proxy/websocket.go @@ -10,25 +10,25 @@ import ( // 当前仅做了转发 websocket 流量 -type WebSocket struct{} +type webSocket struct{} -var DefaultWebSocket WebSocket +var defaultWebSocket webSocket -func (s *WebSocket) WS(conn net.Conn, host string) { - log := log.WithField("in", "WebSocket.WS").WithField("host", host) +func (s *webSocket) ws(conn net.Conn, host string) { + log := log.WithField("in", "webSocket.ws").WithField("host", host) defer conn.Close() remoteConn, err := net.Dial("tcp", host) if err != nil { - LogErr(log, err) + logErr(log, err) return } defer remoteConn.Close() transfer(log, conn, remoteConn) } -func (s *WebSocket) WSS(res http.ResponseWriter, req *http.Request) { - log := log.WithField("in", "WebSocket.WSS").WithField("host", req.Host) +func (s *webSocket) wss(res http.ResponseWriter, req *http.Request) { + log := log.WithField("in", "webSocket.wss").WithField("host", req.Host) upgradeBuf, err := httputil.DumpRequest(req, false) if err != nil { diff --git a/addon/web/client/.eslintignore b/web/client/.eslintignore similarity index 100% rename from addon/web/client/.eslintignore rename to web/client/.eslintignore diff --git a/addon/web/client/.eslintrc.yml b/web/client/.eslintrc.yml similarity index 100% rename from addon/web/client/.eslintrc.yml rename to web/client/.eslintrc.yml diff --git a/addon/web/client/.gitignore b/web/client/.gitignore similarity index 100% rename from addon/web/client/.gitignore rename to web/client/.gitignore diff --git a/addon/web/client/README.md b/web/client/README.md similarity index 100% rename from addon/web/client/README.md rename to web/client/README.md diff --git a/addon/web/client/build/asset-manifest.json b/web/client/build/asset-manifest.json similarity index 100% rename from addon/web/client/build/asset-manifest.json rename to web/client/build/asset-manifest.json diff --git a/addon/web/client/build/favicon.ico b/web/client/build/favicon.ico similarity index 100% rename from addon/web/client/build/favicon.ico rename to web/client/build/favicon.ico diff --git a/addon/web/client/build/index.html b/web/client/build/index.html similarity index 100% rename from addon/web/client/build/index.html rename to web/client/build/index.html diff --git a/addon/web/client/build/logo192.png b/web/client/build/logo192.png similarity index 100% rename from addon/web/client/build/logo192.png rename to web/client/build/logo192.png diff --git a/addon/web/client/build/logo512.png b/web/client/build/logo512.png similarity index 100% rename from addon/web/client/build/logo512.png rename to web/client/build/logo512.png diff --git a/addon/web/client/build/manifest.json b/web/client/build/manifest.json similarity index 100% rename from addon/web/client/build/manifest.json rename to web/client/build/manifest.json diff --git a/addon/web/client/build/robots.txt b/web/client/build/robots.txt similarity index 100% rename from addon/web/client/build/robots.txt rename to web/client/build/robots.txt diff --git a/addon/web/client/build/static/css/2.4659568d.chunk.css b/web/client/build/static/css/2.4659568d.chunk.css similarity index 100% rename from addon/web/client/build/static/css/2.4659568d.chunk.css rename to web/client/build/static/css/2.4659568d.chunk.css diff --git a/addon/web/client/build/static/css/2.4659568d.chunk.css.map b/web/client/build/static/css/2.4659568d.chunk.css.map similarity index 100% rename from addon/web/client/build/static/css/2.4659568d.chunk.css.map rename to web/client/build/static/css/2.4659568d.chunk.css.map diff --git a/addon/web/client/build/static/css/main.9aa5bcb2.chunk.css b/web/client/build/static/css/main.9aa5bcb2.chunk.css similarity index 100% rename from addon/web/client/build/static/css/main.9aa5bcb2.chunk.css rename to web/client/build/static/css/main.9aa5bcb2.chunk.css diff --git a/addon/web/client/build/static/css/main.9aa5bcb2.chunk.css.map b/web/client/build/static/css/main.9aa5bcb2.chunk.css.map similarity index 100% rename from addon/web/client/build/static/css/main.9aa5bcb2.chunk.css.map rename to web/client/build/static/css/main.9aa5bcb2.chunk.css.map diff --git a/addon/web/client/build/static/js/2.90d996cd.chunk.js b/web/client/build/static/js/2.90d996cd.chunk.js similarity index 100% rename from addon/web/client/build/static/js/2.90d996cd.chunk.js rename to web/client/build/static/js/2.90d996cd.chunk.js diff --git a/addon/web/client/build/static/js/2.90d996cd.chunk.js.LICENSE.txt b/web/client/build/static/js/2.90d996cd.chunk.js.LICENSE.txt similarity index 100% rename from addon/web/client/build/static/js/2.90d996cd.chunk.js.LICENSE.txt rename to web/client/build/static/js/2.90d996cd.chunk.js.LICENSE.txt diff --git a/addon/web/client/build/static/js/2.90d996cd.chunk.js.map b/web/client/build/static/js/2.90d996cd.chunk.js.map similarity index 100% rename from addon/web/client/build/static/js/2.90d996cd.chunk.js.map rename to web/client/build/static/js/2.90d996cd.chunk.js.map diff --git a/addon/web/client/build/static/js/3.fdc4294f.chunk.js b/web/client/build/static/js/3.fdc4294f.chunk.js similarity index 100% rename from addon/web/client/build/static/js/3.fdc4294f.chunk.js rename to web/client/build/static/js/3.fdc4294f.chunk.js diff --git a/addon/web/client/build/static/js/3.fdc4294f.chunk.js.map b/web/client/build/static/js/3.fdc4294f.chunk.js.map similarity index 100% rename from addon/web/client/build/static/js/3.fdc4294f.chunk.js.map rename to web/client/build/static/js/3.fdc4294f.chunk.js.map diff --git a/addon/web/client/build/static/js/main.dab9e469.chunk.js b/web/client/build/static/js/main.dab9e469.chunk.js similarity index 100% rename from addon/web/client/build/static/js/main.dab9e469.chunk.js rename to web/client/build/static/js/main.dab9e469.chunk.js diff --git a/addon/web/client/build/static/js/main.dab9e469.chunk.js.map b/web/client/build/static/js/main.dab9e469.chunk.js.map similarity index 100% rename from addon/web/client/build/static/js/main.dab9e469.chunk.js.map rename to web/client/build/static/js/main.dab9e469.chunk.js.map diff --git a/addon/web/client/build/static/js/runtime-main.476c72c1.js b/web/client/build/static/js/runtime-main.476c72c1.js similarity index 100% rename from addon/web/client/build/static/js/runtime-main.476c72c1.js rename to web/client/build/static/js/runtime-main.476c72c1.js diff --git a/addon/web/client/build/static/js/runtime-main.476c72c1.js.map b/web/client/build/static/js/runtime-main.476c72c1.js.map similarity index 100% rename from addon/web/client/build/static/js/runtime-main.476c72c1.js.map rename to web/client/build/static/js/runtime-main.476c72c1.js.map diff --git a/addon/web/client/package.json b/web/client/package.json similarity index 100% rename from addon/web/client/package.json rename to web/client/package.json diff --git a/addon/web/client/public/favicon.ico b/web/client/public/favicon.ico similarity index 100% rename from addon/web/client/public/favicon.ico rename to web/client/public/favicon.ico diff --git a/addon/web/client/public/index.html b/web/client/public/index.html similarity index 100% rename from addon/web/client/public/index.html rename to web/client/public/index.html diff --git a/addon/web/client/public/logo192.png b/web/client/public/logo192.png similarity index 100% rename from addon/web/client/public/logo192.png rename to web/client/public/logo192.png diff --git a/addon/web/client/public/logo512.png b/web/client/public/logo512.png similarity index 100% rename from addon/web/client/public/logo512.png rename to web/client/public/logo512.png diff --git a/addon/web/client/public/manifest.json b/web/client/public/manifest.json similarity index 100% rename from addon/web/client/public/manifest.json rename to web/client/public/manifest.json diff --git a/addon/web/client/public/robots.txt b/web/client/public/robots.txt similarity index 100% rename from addon/web/client/public/robots.txt rename to web/client/public/robots.txt diff --git a/addon/web/client/src/App.css b/web/client/src/App.css similarity index 100% rename from addon/web/client/src/App.css rename to web/client/src/App.css diff --git a/addon/web/client/src/App.test.tsx b/web/client/src/App.test.tsx similarity index 100% rename from addon/web/client/src/App.test.tsx rename to web/client/src/App.test.tsx diff --git a/addon/web/client/src/App.tsx b/web/client/src/App.tsx similarity index 100% rename from addon/web/client/src/App.tsx rename to web/client/src/App.tsx diff --git a/addon/web/client/src/components/BreakPoint.tsx b/web/client/src/components/BreakPoint.tsx similarity index 100% rename from addon/web/client/src/components/BreakPoint.tsx rename to web/client/src/components/BreakPoint.tsx diff --git a/addon/web/client/src/components/EditFlow.tsx b/web/client/src/components/EditFlow.tsx similarity index 100% rename from addon/web/client/src/components/EditFlow.tsx rename to web/client/src/components/EditFlow.tsx diff --git a/addon/web/client/src/components/FlowPreview.tsx b/web/client/src/components/FlowPreview.tsx similarity index 100% rename from addon/web/client/src/components/FlowPreview.tsx rename to web/client/src/components/FlowPreview.tsx diff --git a/addon/web/client/src/components/ViewFlow.tsx b/web/client/src/components/ViewFlow.tsx similarity index 100% rename from addon/web/client/src/components/ViewFlow.tsx rename to web/client/src/components/ViewFlow.tsx diff --git a/addon/web/client/src/flow.ts b/web/client/src/flow.ts similarity index 100% rename from addon/web/client/src/flow.ts rename to web/client/src/flow.ts diff --git a/addon/web/client/src/index.tsx b/web/client/src/index.tsx similarity index 100% rename from addon/web/client/src/index.tsx rename to web/client/src/index.tsx diff --git a/addon/web/client/src/message.ts b/web/client/src/message.ts similarity index 100% rename from addon/web/client/src/message.ts rename to web/client/src/message.ts diff --git a/addon/web/client/src/react-app-env.d.ts b/web/client/src/react-app-env.d.ts similarity index 100% rename from addon/web/client/src/react-app-env.d.ts rename to web/client/src/react-app-env.d.ts diff --git a/addon/web/client/src/reportWebVitals.ts b/web/client/src/reportWebVitals.ts similarity index 100% rename from addon/web/client/src/reportWebVitals.ts rename to web/client/src/reportWebVitals.ts diff --git a/addon/web/client/src/setupTests.ts b/web/client/src/setupTests.ts similarity index 100% rename from addon/web/client/src/setupTests.ts rename to web/client/src/setupTests.ts diff --git a/addon/web/client/src/utils.ts b/web/client/src/utils.ts similarity index 100% rename from addon/web/client/src/utils.ts rename to web/client/src/utils.ts diff --git a/addon/web/client/tsconfig.json b/web/client/tsconfig.json similarity index 100% rename from addon/web/client/tsconfig.json rename to web/client/tsconfig.json diff --git a/addon/web/client/yarn.lock b/web/client/yarn.lock similarity index 100% rename from addon/web/client/yarn.lock rename to web/client/yarn.lock diff --git a/addon/web/conn.go b/web/conn.go similarity index 90% rename from addon/web/conn.go rename to web/conn.go index f9f730a..528b9ff 100644 --- a/addon/web/conn.go +++ b/web/conn.go @@ -5,7 +5,7 @@ import ( "sync" "github.com/gorilla/websocket" - "github.com/lqqyt2423/go-mitmproxy/flow" + "github.com/lqqyt2423/go-mitmproxy/proxy" ) type breakPointRule struct { @@ -31,7 +31,7 @@ func newConn(c *websocket.Conn) *concurrentConn { } } -func (c *concurrentConn) writeMessage(msg *messageFlow, f *flow.Flow) { +func (c *concurrentConn) writeMessage(msg *messageFlow, f *proxy.Flow) { if c.isIntercpt(f, msg) { msg.waitIntercept = 1 } @@ -94,7 +94,7 @@ func (c *concurrentConn) initWaitChan(key string) chan interface{} { } // 是否拦截 -func (c *concurrentConn) isIntercpt(f *flow.Flow, after *messageFlow) bool { +func (c *concurrentConn) isIntercpt(f *proxy.Flow, after *messageFlow) bool { if after.mType != messageTypeRequestBody && after.mType != messageTypeResponseBody { return false } @@ -129,13 +129,13 @@ func (c *concurrentConn) isIntercpt(f *flow.Flow, after *messageFlow) bool { } // 拦截 -func (c *concurrentConn) waitIntercept(f *flow.Flow, after *messageFlow) { +func (c *concurrentConn) waitIntercept(f *proxy.Flow, after *messageFlow) { ch := c.initWaitChan(f.Id.String()) msg := (<-ch).(*messageEdit) // drop if msg.mType == messageTypeDropRequest || msg.mType == messageTypeDropResponse { - f.Response = &flow.Response{ + f.Response = &proxy.Response{ StatusCode: 502, } return diff --git a/addon/web/message.go b/web/message.go similarity index 96% rename from addon/web/message.go rename to web/message.go index a6d2b02..68fd572 100644 --- a/addon/web/message.go +++ b/web/message.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" - "github.com/lqqyt2423/go-mitmproxy/flow" + "github.com/lqqyt2423/go-mitmproxy/proxy" uuid "github.com/satori/go.uuid" ) @@ -74,7 +74,7 @@ type messageFlow struct { content []byte } -func newMessageFlow(mType messageType, f *flow.Flow) *messageFlow { +func newMessageFlow(mType messageType, f *proxy.Flow) *messageFlow { var content []byte var err error = nil @@ -114,8 +114,8 @@ func (m *messageFlow) bytes() []byte { type messageEdit struct { mType messageType id uuid.UUID - request *flow.Request - response *flow.Response + request *proxy.Request + response *proxy.Response } func parseMessageEdit(data []byte) *messageEdit { @@ -158,7 +158,7 @@ func parseMessageEdit(data []byte) *messageEdit { bodyContent := data[42+hl+4:] if mType == messageTypeChangeRequest { - req := new(flow.Request) + req := new(proxy.Request) err := json.Unmarshal(headerContent, req) if err != nil { return nil @@ -166,7 +166,7 @@ func parseMessageEdit(data []byte) *messageEdit { req.Body = bodyContent msg.request = req } else if mType == messageTypeChangeResponse { - res := new(flow.Response) + res := new(proxy.Response) err := json.Unmarshal(headerContent, res) if err != nil { return nil diff --git a/addon/web/web.go b/web/web.go similarity index 85% rename from addon/web/web.go rename to web/web.go index 1c73236..1f5c711 100644 --- a/addon/web/web.go +++ b/web/web.go @@ -7,8 +7,7 @@ import ( "sync" "github.com/gorilla/websocket" - "github.com/lqqyt2423/go-mitmproxy/addon" - "github.com/lqqyt2423/go-mitmproxy/flow" + "github.com/lqqyt2423/go-mitmproxy/proxy" _log "github.com/sirupsen/logrus" ) @@ -17,25 +16,8 @@ var log = _log.WithField("at", "web addon") //go:embed client/build var assets embed.FS -func (web *WebAddon) echo(w http.ResponseWriter, r *http.Request) { - c, err := web.upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - - conn := newConn(c) - web.addConn(conn) - defer func() { - web.removeConn(conn) - c.Close() - }() - - conn.readloop() -} - type WebAddon struct { - addon.Base + proxy.BaseAddon upgrader *websocket.Upgrader conns []*concurrentConn @@ -72,6 +54,23 @@ func NewWebAddon(addr string) *WebAddon { return web } +func (web *WebAddon) echo(w http.ResponseWriter, r *http.Request) { + c, err := web.upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + + conn := newConn(c) + web.addConn(conn) + defer func() { + web.removeConn(conn) + c.Close() + }() + + conn.readloop() +} + func (web *WebAddon) addConn(c *concurrentConn) { web.connsMu.Lock() web.conns = append(web.conns, c) @@ -96,7 +95,7 @@ func (web *WebAddon) removeConn(conn *concurrentConn) { web.conns = append(web.conns[:index], web.conns[index+1:]...) } -func (web *WebAddon) sendFlow(f *flow.Flow, msgFn func() *messageFlow) bool { +func (web *WebAddon) sendFlow(f *proxy.Flow, msgFn func() *messageFlow) bool { web.connsMu.RLock() conns := web.conns web.connsMu.RUnlock() @@ -113,25 +112,25 @@ func (web *WebAddon) sendFlow(f *flow.Flow, msgFn func() *messageFlow) bool { return true } -func (web *WebAddon) Requestheaders(f *flow.Flow) { +func (web *WebAddon) Requestheaders(f *proxy.Flow) { web.sendFlow(f, func() *messageFlow { return newMessageFlow(messageTypeRequest, f) }) } -func (web *WebAddon) Request(f *flow.Flow) { +func (web *WebAddon) Request(f *proxy.Flow) { web.sendFlow(f, func() *messageFlow { return newMessageFlow(messageTypeRequestBody, f) }) } -func (web *WebAddon) Responseheaders(f *flow.Flow) { +func (web *WebAddon) Responseheaders(f *proxy.Flow) { web.sendFlow(f, func() *messageFlow { return newMessageFlow(messageTypeResponse, f) }) } -func (web *WebAddon) Response(f *flow.Flow) { +func (web *WebAddon) Response(f *proxy.Flow) { web.sendFlow(f, func() *messageFlow { return newMessageFlow(messageTypeResponseBody, f) })