fix(git)!: use internal proxy for all git operations (#39426)

Introduces gitproxy module which spawns a small forward proxy as scanner
for git calls
Replaces hostmatcher with matchlist which supports port rules
Deprecates ALLOWED_DOMAINS/BLOCKED_DOMAINS and ALLOW_LOCALNETWORKS
settings in migration in favor of full names we have in security
configs.
Removes `external` preset in favor of lax/strict modes, strict mode
requiring explicit ports if they aren't standard http/s ones.

Breaking changes:
- `external` preset no longer works as deny rule. To enforce that, use
`strict` mode and allow ranges to connect to
- Wildcards are no longer accepted in IP addresses
- `*` is no longer allowed as entry in lists
- domain rules now use curl like syntax `*.example.com` matching
subdomains but not `example.com`, `example.com` matching itself and all
subdomains. `example.*` is not a valid rule
- In the default `lax` mode, `[security] ALLOWED_HOST_LIST` no longer
restricts public hosts, set `EGRESS_MODE = strict` to keep an exclusive
list. A startup warning flags this
- Invalid list entries are logged at startup, invalid
`BLOCKED_HOST_LIST`/`BLOCKED_DOMAINS` entries stop it

Docs: https://gitea.com/gitea/docs/pulls/557
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/webhook/deliver.go

@@ -16,17 +16,14 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
 	user_model "gitea.dev/models/user"
 	webhook_model "gitea.dev/models/webhook"
-	"gitea.dev/modules/glob"
+	"gitea.dev/modules/egress"
 	"gitea.dev/modules/graceful"
-	"gitea.dev/modules/hostmatcher"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/queue"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -271,52 +268,15 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
 	return nil
 }
 
-var (
-	webhookHTTPClient *http.Client
-	once              sync.Once
-	hostMatchers      []glob.Glob
-)
-
-func webhookProxy(allowList *hostmatcher.HostMatchList) func(req *http.Request) (*url.URL, error) {
-	if setting.Webhook.ProxyURL == "" {
-		return proxy.Proxy()
-	}
-
-	once.Do(func() {
-		for _, h := range setting.Webhook.ProxyHosts {
-			if g, err := glob.Compile(h); err == nil {
-				hostMatchers = append(hostMatchers, g)
-			} else {
-				log.Error("glob.Compile %s failed: %v", h, err)
-			}
-		}
-	})
-
-	return func(req *http.Request) (*url.URL, error) {
-		for _, v := range hostMatchers {
-			if v.Match(req.URL.Host) {
-				if !allowList.MatchHostName(req.URL.Host) {
-					return nil, fmt.Errorf("webhook can only call allowed HTTP servers (check your %s setting), deny '%s'", allowList.SettingKeyHint, req.URL.Host)
-				}
-				return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req)
-			}
-		}
-		return http.ProxyFromEnvironment(req)
-	}
-}
+var webhookHTTPClient *http.Client
 
 // Init starts the hooks delivery thread
 func Init() error {
 	timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
-	allowedHostMatcher := hostmatcher.ParseHostMatchList("security.ALLOWED_HOST_LIST", setting.Webhook.AllowedHostList)
 
-	// NewHTTPTransport enforces the allow-list on direct connections; when webhookProxy routes a request
-	// through a configured proxy, restricting the proxied target is the proxy server's responsibility.
-	webhookHTTPClient = &http.Client{
-		Timeout: timeout,
-		Transport: hostmatcher.NewHTTPTransport("webhook", allowedHostMatcher, nil, webhookProxy(allowedHostMatcher), setting.Webhook.ProxyURLFixed,
-			&tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}),
-	}
+	transport := egress.NewWebhookPolicy().NewHTTPTransport()
+	transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}
+	webhookHTTPClient = &http.Client{Timeout: timeout, Transport: transport}
 
 	hookQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "webhook_sender", handler)
 	if hookQueue == nil {

services/webhook/deliver_test.go

@@ -7,82 +7,18 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"strings"
 	"testing"
 	"time"
 
 	"gitea.dev/models/unittest"
 	webhook_model "gitea.dev/models/webhook"
-	"gitea.dev/modules/hostmatcher"
-	"gitea.dev/modules/setting"
 	"gitea.dev/modules/util"
 	webhook_module "gitea.dev/modules/webhook"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-func TestWebhookProxy(t *testing.T) {
-	oldWebhook := setting.Webhook
-	t.Cleanup(func() {
-		setting.Webhook = oldWebhook
-	})
-
-	setting.Webhook.ProxyURL = "http://localhost:8080"
-	setting.Webhook.ProxyURLFixed, _ = url.Parse(setting.Webhook.ProxyURL)
-	setting.Webhook.ProxyHosts = []string{"*.discordapp.com", "discordapp.com"}
-
-	allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", "discordapp.com,s.discordapp.com")
-
-	tests := []struct {
-		req     string
-		want    string
-		wantErr bool
-	}{
-		{
-			req:     "https://discordapp.com/api/webhooks/xxxxxxxxx/xxxxxxxxxxxxxxxxxxx",
-			want:    "http://localhost:8080",
-			wantErr: false,
-		},
-		{
-			req:     "http://s.discordapp.com/assets/xxxxxx",
-			want:    "http://localhost:8080",
-			wantErr: false,
-		},
-		{
-			req:     "http://github.com/a/b",
-			want:    "",
-			wantErr: false,
-		},
-		{
-			req:     "http://www.discordapp.com/assets/xxxxxx",
-			want:    "",
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.req, func(t *testing.T) {
-			req, err := http.NewRequest(http.MethodPost, tt.req, nil)
-			require.NoError(t, err)
-
-			u, err := webhookProxy(allowedHostMatcher)(req)
-			if tt.wantErr {
-				assert.Error(t, err)
-				return
-			}
-
-			assert.NoError(t, err)
-
-			got := ""
-			if u != nil {
-				got = u.String()
-			}
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
 func TestWebhookDeliverAuthorizationHeader(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 

services/webhook/main_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"gitea.dev/models/unittest"
-	"gitea.dev/modules/hostmatcher"
 	"gitea.dev/modules/setting"
 
 	_ "gitea.dev/models"
@@ -17,9 +16,7 @@ import (
 func TestMain(m *testing.M) {
 	unittest.MainTest(m, &unittest.TestOptions{
 		SetUp: func() error {
-			// for tests, allow only loopback IPs. This must run after the test config is loaded (which
-			// resets the shared Security.AllowedHostList) and before Init() builds the delivery client.
-			setting.Security.AllowedHostList = hostmatcher.MatchBuiltinLoopback
+			setting.Webhook.AllowedHostList = "loopback:*"
 			setting.LoadQueueSettings()
 			return Init()
 		},