fix(git)!: use internal proxy for all git operations (#39426)

Introduces gitproxy module which spawns a small forward proxy as scanner
for git calls
Replaces hostmatcher with matchlist which supports port rules
Deprecates ALLOWED_DOMAINS/BLOCKED_DOMAINS and ALLOW_LOCALNETWORKS
settings in migration in favor of full names we have in security
configs.
Removes `external` preset in favor of lax/strict modes, strict mode
requiring explicit ports if they aren't standard http/s ones.

Breaking changes:
- `external` preset no longer works as deny rule. To enforce that, use
`strict` mode and allow ranges to connect to
- Wildcards are no longer accepted in IP addresses
- `*` is no longer allowed as entry in lists
- domain rules now use curl like syntax `*.example.com` matching
subdomains but not `example.com`, `example.com` matching itself and all
subdomains. `example.*` is not a valid rule
- In the default `lax` mode, `[security] ALLOWED_HOST_LIST` no longer
restricts public hosts, set `EGRESS_MODE = strict` to keep an exclusive
list. A startup warning flags this
- Invalid list entries are logged at startup, invalid
`BLOCKED_HOST_LIST`/`BLOCKED_DOMAINS` entries stop it

Docs: https://gitea.com/gitea/docs/pulls/557
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/gitproxy/server.go

@@ -0,0 +1,397 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gitproxy
+
+import (
+	"bufio"
+	"cmp"
+	"context"
+	"crypto/rand"
+	"crypto/subtle"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/netip"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"gitea.dev/modules/egress"
+	"gitea.dev/modules/egress/policy"
+	"gitea.dev/modules/git/gitcmd"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/setting"
+
+	"github.com/Azure/go-ntlmssp"
+	"golang.org/x/net/proxy"
+)
+
+const (
+	proxyURLEnv  = "GITEA_GIT_PROXY" // tells this binary it runs as git's GIT_PROXY_COMMAND
+	directHeader = "X-Gitea-Direct"  // asks for a CONNECT tunnel that skips the operator's proxy, as git:// remotes never used one
+)
+
+// proxyDialer reaches the operator's proxies, which are configuration rather than user input
+var proxyDialer = &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
+
+// copyBuffers beats the reverse proxy's 32 KiB default as it flushes every write, the transport's ReadBufferSize lets a read span chunks
+type copyBuffers struct{}
+
+func (copyBuffers) Get() []byte { return make([]byte, 256*1024) }
+func (copyBuffers) Put([]byte)  {}
+
+// server is a forward proxy for git's remotes that enforces an egress policy on its direct dials.
+type server struct {
+	auth         string // the Proxy-Authorization header git must send
+	policy       *policy.Policy
+	dial         func(ctx context.Context, network, addr string) (net.Conn, error)
+	reverseProxy *httputil.ReverseProxy
+	proxyTLS     *tls.Config
+	proxyNTLM    bool // CONNECT only, the transport can't pin the connection NTLM authenticates
+}
+
+func newServer(p *policy.Policy, auth string, proxyTLS *tls.Config) *server {
+	s := &server{auth: auth, policy: p, dial: p.NewDialContext(), proxyTLS: cmp.Or(proxyTLS, &tls.Config{})}
+	transport := p.NewHTTPTransport()
+	transport.Proxy = s.upstreamProxy
+	transport.TLSClientConfig = s.proxyTLS.Clone() // the transport adds its ALPN protocols to the config it gets
+	transport.ReadBufferSize = 64 * 1024
+	s.reverseProxy = &httputil.ReverseProxy{
+		Rewrite:       func(*httputil.ProxyRequest) {},
+		Transport:     transport,
+		FlushInterval: -1,
+		BufferPool:    copyBuffers{},
+		ErrorHandler:  func(w http.ResponseWriter, _ *http.Request, err error) { writeUpstreamError(w, err) },
+	}
+	return s
+}
+
+// Run routes git's network remotes through a proxy on a random loopback port until ctx is done.
+func Run(ctx context.Context) error {
+	gitPolicy, err := egress.NewGitPolicy()
+	if err != nil {
+		return fmt.Errorf("git proxy: %w", err)
+	}
+	gitOption := func(key, env string) string { return cmp.Or(os.Getenv(env), setting.GitConfig.GetOption(key)) }
+	proxyTLS, err := proxyTLSConfig(gitOption("http.proxySSLCAInfo", "GIT_PROXY_SSL_CAINFO"),
+		gitOption("http.proxySSLCert", "GIT_PROXY_SSL_CERT"), gitOption("http.proxySSLKey", "GIT_PROXY_SSL_KEY"))
+	if err != nil {
+		return fmt.Errorf("git proxy: %w", err)
+	}
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return fmt.Errorf("git proxy: %w", err)
+	}
+	user := url.UserPassword("gitea", rand.Text())
+	handler := newServer(gitPolicy, basicAuth(user), proxyTLS)
+	handler.proxyNTLM = strings.EqualFold(gitOption("http.proxyAuthMethod", "GIT_HTTP_PROXY_AUTHMETHOD"), "ntlm")
+	srv := &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second}
+	context.AfterFunc(ctx, func() { _ = srv.Close() })
+	go func() {
+		if err := srv.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			log.Error("git proxy: %v", err)
+		}
+	}()
+	gitcmd.SetExtraEnvs(gitEnvs((&url.URL{Scheme: "http", User: user, Host: ln.Addr().String()}).String()))
+	return nil
+}
+
+// gitEnvs route git's http(s) remotes through proxyURL and its git:// remotes through MaybeTunnel, command scope config beats every config file and keeps the credentials out of process listings
+func gitEnvs(proxyURL string) []string {
+	envs := []string{
+		"GIT_CONFIG_PARAMETERS=" + strings.TrimSpace(os.Getenv("GIT_CONFIG_PARAMETERS")+" 'http.proxy="+proxyURL+"'"),
+		"GIT_HTTP_PROXY_AUTHMETHOD=basic",
+		"no_proxy=", "NO_PROXY=", // git honors no_proxy even for a configured proxy
+	}
+	if setting.GitConfig.GetOption("core.gitProxy") == "" { // the operator's own git:// proxy command stays in charge
+		envs = append(envs, "GIT_PROXY_COMMAND="+setting.AppPath, proxyURLEnv+"="+proxyURL)
+	}
+	return envs
+}
+
+// MaybeTunnel serves as git's GIT_PROXY_COMMAND for git:// remotes when git runs this binary with host and port, it returns otherwise
+func MaybeTunnel() {
+	proxyURL := os.Getenv(proxyURLEnv)
+	if proxyURL == "" || len(os.Args) != 3 {
+		return
+	}
+	if err := tunnel(proxyURL, net.JoinHostPort(os.Args[1], os.Args[2])); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func tunnel(proxyURL, target string) error {
+	gitProxy, err := url.Parse(proxyURL)
+	if err != nil {
+		return err
+	}
+	conn, err := net.Dial("tcp", gitProxy.Host)
+	if err != nil {
+		return err
+	}
+	if conn, err = openTunnel(context.Background(), conn, gitProxy, target, http.Header{directHeader: {"1"}}, false); err != nil {
+		return err
+	}
+	go func() {
+		_, _ = io.Copy(conn, os.Stdin)
+		closeWrite(conn)
+	}()
+	_, err = io.Copy(os.Stdout, conn)
+	return err
+}
+
+// proxyTLSConfig loads the files of git's http.proxySSL* options, like curl the CA file replaces the system roots
+func proxyTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) {
+	cfg := &tls.Config{}
+	if caFile != "" {
+		pemData, err := os.ReadFile(caFile)
+		if err != nil {
+			return nil, err
+		}
+		cfg.RootCAs = x509.NewCertPool()
+		if !cfg.RootCAs.AppendCertsFromPEM(pemData) {
+			return nil, fmt.Errorf("no certificates in %s", caFile)
+		}
+	}
+	if certFile != "" {
+		cert, err := tls.LoadX509KeyPair(certFile, cmp.Or(keyFile, certFile))
+		if err != nil {
+			return nil, err
+		}
+		cfg.Certificates = []tls.Certificate{cert}
+	}
+	return cfg, nil
+}
+
+func basicAuth(user *url.Userinfo) string {
+	password, _ := user.Password()
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(user.Username()+":"+password))
+}
+
+func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if subtle.ConstantTimeCompare([]byte(r.Header.Get("Proxy-Authorization")), []byte(s.auth)) != 1 {
+		w.Header().Set("Proxy-Authenticate", `Basic realm="gitea egress proxy"`)
+		http.Error(w, "egress: proxy authentication required", http.StatusProxyAuthRequired)
+		return
+	}
+	switch {
+	case r.Method == http.MethodConnect:
+		s.handleConnect(w, r)
+	case r.URL.Scheme == "http" && r.URL.Host != "":
+		s.reverseProxy.ServeHTTP(w, r)
+	default:
+		http.Error(w, "egress: CONNECT or an absolute http URI required", http.StatusBadRequest)
+	}
+}
+
+func (s *server) handleConnect(w http.ResponseWriter, r *http.Request) {
+	if host, _, err := net.SplitHostPort(r.URL.Host); err != nil || host == "" {
+		http.Error(w, "egress: invalid CONNECT target", http.StatusBadRequest)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
+	defer cancel()
+	dial := s.dialUpstream
+	if r.Header.Get(directHeader) != "" {
+		dial = func(ctx context.Context, target string) (net.Conn, error) { return s.dial(ctx, "tcp", target) }
+	}
+	upstream, err := dial(ctx, r.URL.Host)
+	if err != nil {
+		writeUpstreamError(w, err)
+		return
+	}
+	client, buffered, err := http.NewResponseController(w).Hijack()
+	if err != nil {
+		_ = upstream.Close()
+		http.Error(w, "egress: "+err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
+		_ = client.Close()
+		_ = upstream.Close()
+		return
+	}
+	relay(withBuffered(client, buffered.Reader), upstream)
+}
+
+// upstreamProxy selects the operator's proxy for req, local targets are dialed directly as they would name the proxy's own host
+func (s *server) upstreamProxy(req *http.Request) (proxyURL *url.URL, err error) {
+	if !isLocalHost(req.URL.Hostname()) {
+		proxyURL, err = s.policy.Proxy(req)
+	}
+	return proxyURL, err
+}
+
+func isLocalHost(host string) bool {
+	host = strings.TrimSuffix(strings.ToLower(host), ".")
+	if host == "localhost" || strings.HasSuffix(host, ".localhost") {
+		return true
+	}
+	ip, err := netip.ParseAddr(host)
+	return err == nil && (ip.Unmap().IsLoopback() || ip.IsUnspecified())
+}
+
+func (s *server) dialUpstream(ctx context.Context, target string) (net.Conn, error) {
+	proxyURL, err := s.upstreamProxy(&http.Request{URL: &url.URL{Scheme: "https", Host: strings.TrimSuffix(target, ":443")}})
+	switch {
+	case err != nil:
+		return nil, err
+	case proxyURL == nil:
+		return s.dial(ctx, "tcp", target)
+	case proxyURL.Scheme == "socks5" || proxyURL.Scheme == "socks5h":
+		dialer, err := proxy.FromURL(proxyURL, proxyDialer)
+		if err != nil {
+			return nil, err
+		}
+		ctxDialer, ok := dialer.(proxy.ContextDialer)
+		if !ok {
+			return nil, errors.New("egress: socks dialer lacks context support")
+		}
+		return ctxDialer.DialContext(ctx, "tcp", target)
+	default:
+		return s.connectVia(ctx, proxyURL, target)
+	}
+}
+
+func (s *server) connectVia(ctx context.Context, proxyURL *url.URL, target string) (net.Conn, error) {
+	conn, err := proxyDialer.DialContext(ctx, "tcp", policy.ProxyDialAddr(proxyURL))
+	if err != nil {
+		return nil, err
+	}
+	if proxyURL.Scheme == "https" {
+		cfg := s.proxyTLS.Clone()
+		cfg.ServerName = proxyURL.Hostname()
+		conn = tls.Client(conn, cfg)
+	}
+	return openTunnel(ctx, conn, proxyURL, target, http.Header{}, s.proxyNTLM)
+}
+
+// openTunnel opens a CONNECT tunnel to target over conn to the proxy at proxyURL, closing conn on failure
+func openTunnel(ctx context.Context, conn net.Conn, proxyURL *url.URL, target string, header http.Header, ntlm bool) (_ net.Conn, err error) {
+	defer func() {
+		if err != nil {
+			_ = conn.Close()
+		}
+	}()
+	defer context.AfterFunc(ctx, func() { _ = conn.Close() })()
+	switch {
+	case ntlm:
+		negotiate, _ := ntlmssp.NewNegotiateMessage("", "")
+		header.Set("Proxy-Authorization", "NTLM "+base64.StdEncoding.EncodeToString(negotiate))
+	case proxyURL.User != nil:
+		header.Set("Proxy-Authorization", basicAuth(proxyURL.User))
+	}
+	req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: header}
+	reader := bufio.NewReader(conn)
+	resp, err := roundTrip(conn, reader, req)
+	if err != nil {
+		return nil, err
+	}
+	if ntlm && resp.StatusCode == http.StatusProxyAuthRequired {
+		_ = resp.Body.Close() // drains it for the next request on the connection
+		authenticate, err := ntlmAuthenticate(resp.Header, proxyURL.User)
+		if err != nil {
+			return nil, err
+		}
+		header.Set("Proxy-Authorization", authenticate)
+		if resp, err = roundTrip(conn, reader, req); err != nil {
+			return nil, err
+		}
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
+		return nil, fmt.Errorf("egress: proxy refused CONNECT: %s", strings.TrimSpace(resp.Status+" "+string(body)))
+	}
+	return withBuffered(conn, reader), nil
+}
+
+func roundTrip(conn net.Conn, reader *bufio.Reader, req *http.Request) (*http.Response, error) {
+	if err := req.Write(conn); err != nil {
+		return nil, err
+	}
+	return http.ReadResponse(reader, req)
+}
+
+func ntlmAuthenticate(header http.Header, user *url.Userinfo) (string, error) {
+	for _, value := range header.Values("Proxy-Authenticate") {
+		if encoded, ok := strings.CutPrefix(value, "NTLM "); ok {
+			challenge, err := base64.StdEncoding.DecodeString(encoded)
+			if err != nil {
+				return "", err
+			}
+			password, _ := user.Password()
+			authenticate, err := ntlmssp.NewAuthenticateMessage(challenge, user.Username(), password, nil)
+			if err != nil {
+				return "", err
+			}
+			return "NTLM " + base64.StdEncoding.EncodeToString(authenticate), nil
+		}
+	}
+	return "", errors.New("egress: proxy sent no NTLM challenge")
+}
+
+func writeUpstreamError(w http.ResponseWriter, err error) {
+	var netErr net.Error
+	switch {
+	case errors.Is(err, policy.ErrDenied):
+		http.Error(w, "egress: target denied by policy", http.StatusForbidden)
+	case errors.As(err, &netErr) && netErr.Timeout():
+		http.Error(w, "egress: upstream timeout: "+err.Error(), http.StatusGatewayTimeout)
+	default:
+		http.Error(w, "egress: upstream failed: "+err.Error(), http.StatusBadGateway)
+	}
+}
+
+// bufferedConn reads the bytes a handshake left buffered before the rest of the connection
+type bufferedConn struct {
+	net.Conn
+	r io.Reader
+}
+
+func (c *bufferedConn) Read(p []byte) (int, error) { return c.r.Read(p) }
+
+func (c *bufferedConn) CloseWrite() error {
+	closeWrite(c.Conn)
+	return nil
+}
+
+func withBuffered(conn net.Conn, r *bufio.Reader) net.Conn {
+	if r.Buffered() == 0 {
+		return conn // a bare socket lets io.Copy splice
+	}
+	return &bufferedConn{Conn: conn, r: r}
+}
+
+// relay copies both ways and passes each end of stream on, the git:// protocol needs the half-close
+func relay(client, upstream net.Conn) {
+	done := make(chan struct{})
+	go func() {
+		_, _ = io.Copy(upstream, client)
+		closeWrite(upstream)
+		close(done)
+	}()
+	_, _ = io.Copy(client, upstream)
+	closeWrite(client)
+	<-done
+	_ = client.Close()
+	_ = upstream.Close()
+}
+
+func closeWrite(conn net.Conn) {
+	if cw, ok := conn.(interface{ CloseWrite() error }); ok {
+		_ = cw.CloseWrite()
+	} else {
+		_ = conn.Close()
+	}
+}

services/gitproxy/server_test.go

@@ -0,0 +1,444 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gitproxy
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"gitea.dev/modules/egress/policy"
+	"gitea.dev/modules/git/gitcmd"
+	"gitea.dev/modules/setting"
+	"gitea.dev/modules/test"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testAuth = "Basic dGVzdDp0ZXN0"
+
+var (
+	allowLoopback = policy.NewPolicy("test", policy.Lax, policy.WithAllow("loopback", ""))
+	blockLoopback = policy.NewPolicy("test", policy.Lax, policy.WithBlock("loopback", ""))
+)
+
+func listen(t *testing.T) net.Listener {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+	return ln
+}
+
+func serveConns(t *testing.T, handle func(net.Conn)) string {
+	ln := listen(t)
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go func() {
+				defer conn.Close()
+				handle(conn)
+			}()
+		}
+	}()
+	return ln.Addr().String()
+}
+
+func startEcho(t *testing.T) string {
+	return serveConns(t, func(conn net.Conn) { _, _ = io.Copy(conn, conn) })
+}
+
+func startProxy(t *testing.T, srv *server) string {
+	proxySrv := httptest.NewServer(srv)
+	t.Cleanup(proxySrv.Close)
+	return proxySrv.Listener.Addr().String()
+}
+
+func viaProxy(u *url.URL) *policy.Policy {
+	return policy.NewPolicy("test", policy.Lax, policy.WithAllow("loopback", ""), policy.WithProxy(http.ProxyURL(u)))
+}
+
+func serve(srv *server, method, target, auth string) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(method, target, nil)
+	req.Header.Set("Proxy-Authorization", auth)
+	rec := httptest.NewRecorder()
+	srv.ServeHTTP(rec, req)
+	return rec
+}
+
+func connect(t *testing.T, proxyAddr, target, auth string) (net.Conn, *bufio.Reader, int) {
+	conn, err := net.Dial("tcp", proxyAddr)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = conn.Close() })
+	_, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\nProxy-Authorization: %s\r\n\r\n", target, target, auth)
+	require.NoError(t, err)
+	br := bufio.NewReader(conn)
+	resp, err := http.ReadResponse(br, &http.Request{Method: http.MethodConnect})
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	return conn, br, resp.StatusCode
+}
+
+func assertEcho(t *testing.T, conn net.Conn, br *bufio.Reader) {
+	_, err := conn.Write([]byte("ping\n"))
+	require.NoError(t, err)
+	reply, err := br.ReadString('\n')
+	require.NoError(t, err)
+	assert.Equal(t, "ping\n", reply)
+}
+
+func startConnectOperator(t *testing.T, useTLS bool, reply string, seen chan<- *http.Request) *httptest.Server {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		seen <- r
+		conn, buf, err := http.NewResponseController(w).Hijack()
+		if err != nil {
+			return
+		}
+		defer conn.Close()
+		_, _ = buf.WriteString(reply)
+		_ = buf.Flush()
+		_, _ = io.Copy(conn, buf)
+	})
+	operator := httptest.NewUnstartedServer(handler)
+	if useTLS {
+		operator.TLS = &tls.Config{ClientAuth: tls.RequireAnyClientCert}
+		operator.StartTLS()
+	} else {
+		operator.Start()
+	}
+	t.Cleanup(operator.Close)
+	return operator
+}
+
+func startSOCKS5(t *testing.T) string {
+	return serveConns(t, func(conn net.Conn) {
+		br := bufio.NewReader(conn)
+		read := func(n int) []byte {
+			buf := make([]byte, n)
+			_, _ = io.ReadFull(br, buf)
+			return buf
+		}
+		_, _ = conn.Write([]byte{5, 2})
+		_ = read(int(read(2)[1]))
+		gotUser := string(read(int(read(2)[1])))
+		if gotPass := string(read(int(read(1)[0]))); gotUser != "user" || gotPass != "secret" {
+			_, _ = conn.Write([]byte{1, 1})
+			return
+		}
+		_, _ = conn.Write([]byte{1, 0})
+		_ = read(int(read(5)[4]) + 2)
+		_, _ = conn.Write([]byte{5, 0, 0, 1, 0, 0, 0, 0, 0, 0})
+		_, _ = io.Copy(conn, br)
+	})
+}
+
+func TestUpstreamProxy(t *testing.T) {
+	t.Parallel()
+	var seen []string
+	errStop := errors.New("stop")
+	s := &server{policy: policy.NewPolicy("test", policy.Lax, policy.WithProxy(func(r *http.Request) (*url.URL, error) {
+		seen = append(seen, r.URL.Host)
+		return nil, errStop
+	}))}
+	for _, target := range []string{"github.com:443", "github.com:8443", "[2001:db8::1]:443"} {
+		_, err := s.dialUpstream(t.Context(), target)
+		assert.ErrorIs(t, err, errStop)
+	}
+	assert.Equal(t, []string{"github.com", "github.com:8443", "[2001:db8::1]"}, seen)
+
+	for _, host := range []string{"localhost", "git.localhost.", "127.0.0.2", "::1", "::ffff:127.0.0.1", "0.0.0.0", "::"} {
+		u, err := s.upstreamProxy(&http.Request{URL: &url.URL{Host: net.JoinHostPort(host, "80")}})
+		assert.NoError(t, err, host)
+		assert.Nil(t, u, host)
+	}
+}
+
+func TestWriteUpstreamError(t *testing.T) {
+	t.Parallel()
+	for err, want := range map[error]int{
+		fmt.Errorf("dial: %w", policy.ErrDenied):                          http.StatusForbidden,
+		context.DeadlineExceeded:                                          http.StatusGatewayTimeout,
+		&net.OpError{Op: "dial", Net: "tcp", Err: os.ErrDeadlineExceeded}: http.StatusGatewayTimeout,
+		errors.New("refused"):                                             http.StatusBadGateway,
+	} {
+		rec := httptest.NewRecorder()
+		writeUpstreamError(rec, err)
+		assert.Equal(t, want, rec.Code, err.Error())
+	}
+}
+
+func TestRelay(t *testing.T) {
+	t.Parallel()
+	client, clientEnd := net.Pipe()
+	upstreamEnd, upstream := net.Pipe()
+	done := make(chan struct{})
+	go func() {
+		relay(clientEnd, upstreamEnd)
+		close(done)
+	}()
+	require.NoError(t, upstream.Close())
+	_, err := client.Read(make([]byte, 1))
+	assert.ErrorIs(t, err, io.EOF)
+	<-done
+}
+
+func TestProxyCONNECT(t *testing.T) {
+	t.Parallel()
+	echo := startEcho(t)
+	srv := newServer(allowLoopback, testAuth, nil)
+
+	assert.Equal(t, http.StatusProxyAuthRequired, serve(srv, http.MethodConnect, echo, "").Code)
+	assert.Equal(t, http.StatusForbidden, serve(newServer(blockLoopback, "", nil), http.MethodConnect, echo, "").Code)
+	assert.Equal(t, http.StatusBadRequest, serve(srv, http.MethodConnect, "127.0.0.1", testAuth).Code)
+
+	conn, br, status := connect(t, startProxy(t, srv), echo, testAuth)
+	require.Equal(t, http.StatusOK, status)
+	assertEcho(t, conn, br)
+}
+
+func TestProxyCONNECTOperator(t *testing.T) {
+	t.Parallel()
+	const target = "git.example.com:443"
+
+	t.Run("HTTP", func(t *testing.T) {
+		t.Parallel()
+		seen := make(chan *http.Request, 1)
+		operator := startConnectOperator(t, false, "HTTP/1.1 204 No Content\r\n\r\nEARLY", seen)
+		opURL, err := url.Parse(operator.URL)
+		require.NoError(t, err)
+		opURL.User = url.UserPassword("user", "secret")
+
+		conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", nil)), target, "")
+		require.Equal(t, http.StatusOK, status)
+		early := make([]byte, 5)
+		_, err = io.ReadFull(br, early)
+		require.NoError(t, err)
+		assert.Equal(t, "EARLY", string(early))
+		assertEcho(t, conn, br)
+		req := <-seen
+		assert.Equal(t, target, req.Host)
+		assert.Equal(t, basicAuth(opURL.User), req.Header.Get("Proxy-Authorization"))
+	})
+
+	t.Run("HTTPS", func(t *testing.T) {
+		t.Parallel()
+		operator := startConnectOperator(t, true, "HTTP/1.1 200 OK\r\n\r\n", make(chan *http.Request, 1))
+		opURL, err := url.Parse(operator.URL)
+		require.NoError(t, err)
+		key, err := x509.MarshalPKCS8PrivateKey(operator.TLS.Certificates[0].PrivateKey)
+		require.NoError(t, err)
+		pemFile := filepath.Join(t.TempDir(), "proxy.pem")
+		pemData := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: operator.Certificate().Raw}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key})...)
+		require.NoError(t, os.WriteFile(pemFile, pemData, 0o600))
+		proxyTLS, err := proxyTLSConfig(pemFile, pemFile, "")
+		require.NoError(t, err)
+
+		conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", proxyTLS)), target, "")
+		require.Equal(t, http.StatusOK, status)
+		assertEcho(t, conn, br)
+	})
+
+	t.Run("SOCKS5", func(t *testing.T) {
+		t.Parallel()
+		opURL := &url.URL{Scheme: "socks5", User: url.UserPassword("user", "secret"), Host: startSOCKS5(t)}
+		conn, br, status := connect(t, startProxy(t, newServer(viaProxy(opURL), "", nil)), target, "")
+		require.Equal(t, http.StatusOK, status)
+		assertEcho(t, conn, br)
+	})
+
+	t.Run("NTLM", func(t *testing.T) {
+		t.Parallel()
+		challenge := make([]byte, 48)
+		copy(challenge, "NTLMSSP\x00")
+		challenge[8] = 2
+		binary.LittleEndian.PutUint32(challenge[20:], 0x201)
+		auths := make(chan string, 2)
+		addr := serveConns(t, func(conn net.Conn) {
+			br := bufio.NewReader(conn)
+			for _, reply := range []string{"HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: NTLM " + base64.StdEncoding.EncodeToString(challenge) + "\r\nContent-Length: 4\r\n\r\ndeny", "HTTP/1.1 200 OK\r\n\r\n"} {
+				req, err := http.ReadRequest(br)
+				if err != nil {
+					return
+				}
+				auths <- req.Header.Get("Proxy-Authorization")
+				_, _ = io.WriteString(conn, reply)
+			}
+		})
+		srv := newServer(viaProxy(&url.URL{Scheme: "http", User: url.UserPassword(`CORP\alice`, "secret"), Host: addr}), "", nil)
+		srv.proxyNTLM = true
+
+		_, _, status := connect(t, startProxy(t, srv), target, "")
+		require.Equal(t, http.StatusOK, status)
+		assert.Regexp(t, "^NTLM TlRMTVNTUAAB", <-auths)
+		assert.Regexp(t, "^NTLM TlRMTVNTUAAD", <-auths)
+	})
+
+	t.Run("Refused", func(t *testing.T) {
+		t.Parallel()
+		operator := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "blocked by policy", http.StatusForbidden)
+		}))
+		t.Cleanup(operator.Close)
+		opURL, err := url.Parse(operator.URL)
+		require.NoError(t, err)
+
+		rec := serve(newServer(viaProxy(opURL), "", nil), http.MethodConnect, target, "")
+		assert.Equal(t, http.StatusBadGateway, rec.Code)
+		assert.Contains(t, rec.Body.String(), "blocked by policy")
+	})
+
+	t.Run("ClientGone", func(t *testing.T) {
+		t.Parallel()
+		ln := listen(t)
+		operatorClosed := make(chan struct{})
+		requestSeen := make(chan struct{})
+		go func() {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			defer close(operatorClosed)
+			defer conn.Close()
+			if _, err := http.ReadRequest(bufio.NewReader(conn)); err != nil {
+				return
+			}
+			close(requestSeen)
+			_, _ = io.Copy(io.Discard, conn)
+		}()
+
+		conn, err := net.Dial("tcp", startProxy(t, newServer(viaProxy(&url.URL{Scheme: "http", Host: ln.Addr().String()}), "", nil)))
+		require.NoError(t, err)
+		_, err = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", target, target)
+		require.NoError(t, err)
+		responded := make(chan struct{})
+		go func() {
+			_, _ = conn.Read(make([]byte, 1))
+			close(responded)
+		}()
+		select {
+		case <-requestSeen:
+		case <-responded:
+			t.Fatal("proxy answered before reaching the operator")
+		}
+		require.NoError(t, conn.Close())
+		<-operatorClosed
+	})
+}
+
+func TestProxyHTTP(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Forward", func(t *testing.T) {
+		t.Parallel()
+		firstRead := make(chan struct{})
+		origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			assert.Empty(t, r.Header.Get("X-Drop-Me"))
+			assert.False(t, r.Close)
+			w.Header().Set("Connection", "X-Origin-Drop")
+			w.Header().Set("X-Origin-Drop", "dropped")
+			_, _ = w.Write([]byte("0008NAK\n"))
+			http.NewResponseController(w).Flush()
+			<-firstRead
+			_, _ = w.Write([]byte("0000"))
+		}))
+		t.Cleanup(origin.Close)
+
+		req, err := http.NewRequest(http.MethodGet, origin.URL+"/git-upload-pack", nil)
+		require.NoError(t, err)
+		req.Header.Set("Connection", "close, X-Drop-Me")
+		req.Header.Set("X-Drop-Me", "dropped")
+		proxyURL := &url.URL{Scheme: "http", Host: startProxy(t, newServer(allowLoopback, "", nil))}
+		resp, err := (&http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}).Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		assert.Empty(t, resp.Header.Get("X-Origin-Drop"))
+
+		buf := make([]byte, 8)
+		_, err = io.ReadFull(resp.Body, buf)
+		require.NoError(t, err)
+		assert.Equal(t, "0008NAK\n", string(buf))
+		close(firstRead)
+		rest, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		assert.Equal(t, "0000", string(rest))
+	})
+
+	t.Run("ViaOperator", func(t *testing.T) {
+		t.Parallel()
+		operator := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write([]byte(r.URL.String()))
+		}))
+		t.Cleanup(operator.Close)
+		opURL, err := url.Parse(operator.URL)
+		require.NoError(t, err)
+
+		rec := serve(newServer(viaProxy(opURL), "", nil), http.MethodGet, "http://git.example.com/repo.git", "")
+		assert.Equal(t, "http://git.example.com/repo.git", rec.Body.String())
+	})
+
+	t.Run("Denied", func(t *testing.T) {
+		t.Parallel()
+		origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			t.Error("the blocked origin must not be reached")
+		}))
+		t.Cleanup(origin.Close)
+		assert.Equal(t, http.StatusForbidden, serve(newServer(blockLoopback, "", nil), http.MethodGet, origin.URL, "").Code)
+	})
+
+	t.Run("OriginForm", func(t *testing.T) {
+		t.Parallel()
+		assert.Equal(t, http.StatusBadRequest, serve(newServer(allowLoopback, "", nil), http.MethodGet, "/", "").Code)
+	})
+}
+
+func TestMain(m *testing.M) {
+	MaybeTunnel()
+	m.Run()
+}
+
+func TestRun(t *testing.T) {
+	base := t.TempDir()
+	exe, err := os.Executable()
+	require.NoError(t, err)
+	repos, err := filepath.Abs("../../modules/git/tests/repos")
+	require.NoError(t, err)
+	defer test.MockVariableValue(&setting.Git.HomePath, base)()
+	defer test.MockVariableValue(&setting.AppPath, exe)()
+	defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "127.0.0.1/32")()
+	t.Cleanup(func() { gitcmd.SetExtraEnvs(nil) })
+	require.NoError(t, Run(t.Context()))
+
+	stdout, _, runErr := gitcmd.NewCommand("config", "--get", "http.proxy").RunStdString(t.Context())
+	require.NoError(t, runErr)
+	assert.Contains(t, stdout, "http://gitea:")
+
+	_, port, err := net.SplitHostPort(serveConns(t, func(conn net.Conn) {
+		daemon := exec.Command("git", "daemon", "--inetd", "--export-all", "--base-path="+repos)
+		daemon.Stdin, daemon.Stdout = conn, conn
+		_ = daemon.Run()
+	}))
+	require.NoError(t, err)
+	require.NoError(t, gitcmd.NewCommand("clone", "-q", "--bare").AddDynamicArguments("git://127.0.0.1:"+port+"/repo1_bare", filepath.Join(base, "allowed")).Run(t.Context()))
+	_, stderr, runErr := gitcmd.NewCommand("clone", "-q", "--bare").AddDynamicArguments("git://127.0.0.2:"+port+"/repo1_bare", filepath.Join(base, "denied")).RunStdString(t.Context())
+	require.Error(t, runErr)
+	assert.Contains(t, stderr, "target denied by policy")
+}

services/migrations/codebase.go

@@ -15,7 +15,6 @@ import (
 
 	"gitea.dev/modules/log"
 	base "gitea.dev/modules/migration"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/structs"
 )
 
@@ -79,6 +78,7 @@ type CodebaseDownloader struct {
 // NewCodebaseDownloader creates a new downloader
 func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repoName, username, password string) *CodebaseDownloader {
 	baseURL, _ := url.Parse("https://api3.codebasehq.com")
+	transport := NewMigrationHTTPTransport()
 
 	downloader := &CodebaseDownloader{
 		baseURL:    baseURL,
@@ -86,14 +86,13 @@ func NewCodebaseDownloader(_ context.Context, projectURL *url.URL, project, repo
 		project:    project,
 		repoName:   repoName,
 		client: &http.Client{
-			Transport: &http.Transport{
-				Proxy: func(req *http.Request) (*url.URL, error) {
-					if len(username) > 0 && len(password) > 0 {
-						req.SetBasicAuth(username, password)
-					}
-					return proxy.Proxy()(req)
-				},
-			},
+			Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+				if username != "" && password != "" {
+					req = req.Clone(req.Context())
+					req.SetBasicAuth(username, password)
+				}
+				return transport.RoundTrip(req)
+			}),
 		},
 		userMap:   make(map[int64]*codebaseUser),
 		commitMap: make(map[string]string),

services/migrations/gitea_uploader.go

@@ -948,7 +948,7 @@ func (g *GiteaLocalUploader) CreateReviews(ctx context.Context, reviews ...*base
 
 // Rollback when migrating failed, this will rollback all the changes.
 func (g *GiteaLocalUploader) Rollback() error {
-	if g.repo != nil && g.repo.ID > 0 {
+	if g.repo != nil && g.repo.ID > 0 && g.gitRepo != nil {
 		g.gitRepo.Close()
 
 		// do not delete the repository, otherwise the end users won't be able to see the last error message

services/migrations/github.go

@@ -17,7 +17,6 @@ import (
 	"gitea.dev/modules/git"
 	"gitea.dev/modules/log"
 	base "gitea.dev/modules/migration"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/structs"
 
 	"github.com/google/go-github/v92/github"
@@ -107,13 +106,8 @@ func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token
 			}
 		}
 	} else {
-		transport := NewMigrationHTTPTransport()
-		transport.Proxy = func(req *http.Request) (*url.URL, error) {
-			req.SetBasicAuth(userName, password)
-			return proxy.Proxy()(req)
-		}
 		client := &http.Client{
-			Transport: transport,
+			Transport: &github.BasicAuthTransport{Transport: NewMigrationHTTPTransport(), Username: userName, Password: password},
 		}
 		if err := downloader.addClient(client, baseURL); err != nil {
 			return nil, err

services/migrations/gogs.go

@@ -107,11 +107,12 @@ func (g *GogsDownloader) client(ctx context.Context) *gogs.Client {
 	gogsClient := gogs.NewClient(g.baseURL, g.token)
 	gogsClient.SetHTTPClient(&http.Client{
 		Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			req = req.Clone(ctx)
 			if g.password != "" {
 				// Gogs client lacks the support for basic auth, this is the only way to set it
 				req.SetBasicAuth(g.userName, g.password)
 			}
-			return httpTransport.RoundTrip(req.WithContext(ctx))
+			return httpTransport.RoundTrip(req)
 		}),
 	})
 	return gogsClient

services/migrations/http_client.go

@@ -6,37 +6,23 @@ package migrations
 import (
 	"crypto/tls"
 	"net/http"
+	"sync"
 
-	"gitea.dev/modules/hostmatcher"
-	"gitea.dev/modules/proxy"
+	"gitea.dev/modules/egress"
 	"gitea.dev/modules/setting"
-	"gitea.dev/modules/util"
 )
 
-// migrationHTTPClient is the shared migration client. Callers that would otherwise build a client per
-// request use it (via getMigrationHTTPClient) so a single connection pool is reused across downloads —
-// e.g. many release assets from the same host — instead of a fresh pool and TLS handshake each time. It
-// is built lazily on first use and reset by Init whenever the allow/block lists change; OnceValue keeps
-// concurrent callers sharing a single client instead of racing to create their own.
-var migrationHTTPClient = util.OnceValue[*http.Client]{Func: newMigrationHTTPClient}
+// getMigrationHTTPClient returns the shared migration client, so downloads from one host reuse its connections
+var getMigrationHTTPClient = sync.OnceValue(newMigrationHTTPClient)
 
 // newMigrationHTTPClient returns a HTTP client for migration
 func newMigrationHTTPClient() *http.Client {
-	return &http.Client{
-		Transport: NewMigrationHTTPTransport(),
-	}
+	return &http.Client{Transport: NewMigrationHTTPTransport()}
 }
 
-// getMigrationHTTPClient returns the shared migration client, building it on first use so no request
-// escapes the SSRF-validated transport even before Init has run.
-func getMigrationHTTPClient() *http.Client {
-	return migrationHTTPClient.Value()
-}
-
-// NewMigrationHTTPTransport returns a HTTP transport for migration. The target is validated against the
-// allow/block lists on both the direct-dial and proxy paths, so a configured proxy cannot be used to
-// reach an otherwise-forbidden target (SSRF).
+// NewMigrationHTTPTransport returns a HTTP transport for migration, enforcing the migration policy on its direct dials.
 func NewMigrationHTTPTransport() *http.Transport {
-	return hostmatcher.NewHTTPTransport("migration", allowList, blockList, proxy.Proxy(), setting.Proxy.ProxyURLFixed,
-		&tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify})
+	t := egress.NewMigrationPolicy().NewHTTPTransport()
+	t.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify}
+	return t
 }

services/migrations/main_test.go

@@ -10,11 +10,13 @@ import (
 
 	"gitea.dev/models/unittest"
 	base "gitea.dev/modules/migration"
+	"gitea.dev/modules/setting"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestMain(m *testing.M) {
+	setting.Migrations.AllowedHostList = "private:*, loopback:*"
 	unittest.MainTest(m)
 }
 

services/migrations/migrate.go

@@ -8,7 +8,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net"
 	"net/url"
 	"path/filepath"
 	"strings"
@@ -17,8 +16,8 @@ import (
 	system_model "gitea.dev/models/system"
 	user_model "gitea.dev/models/user"
 	"gitea.dev/modules/container"
+	"gitea.dev/modules/egress"
 	"gitea.dev/modules/git"
-	"gitea.dev/modules/hostmatcher"
 	"gitea.dev/modules/log"
 	base "gitea.dev/modules/migration"
 	"gitea.dev/modules/setting"
@@ -28,12 +27,7 @@ import (
 // MigrateOptions is equal to base.MigrateOptions
 type MigrateOptions = base.MigrateOptions
 
-var (
-	factories []base.DownloaderFactory
-
-	allowList *hostmatcher.HostMatchList
-	blockList *hostmatcher.HostMatchList
-)
+var factories []base.DownloaderFactory
 
 // RegisterDownloaderFactory registers a downloader factory
 func RegisterDownloaderFactory(factory base.DownloaderFactory) {
@@ -76,33 +70,9 @@ func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error {
 		return &git.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
 	}
 
-	hostName, _, errIgnored := net.SplitHostPort(u.Host)
-	if errIgnored != nil {
-		hostName = u.Host // u.Host can be "host" or "host:port"
+	if err := egress.NewMigrationPolicy().CheckHostIPs(u); err != nil {
+		return &git.ErrInvalidCloneAddr{Host: u.Hostname(), IsPermissionDenied: true}
 	}
-
-	// some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error
-	addrList, _ := net.LookupIP(hostName)
-	return checkByAllowBlockList(hostName, addrList)
-}
-
-func checkByAllowBlockList(hostName string, addrList []net.IP) error {
-	ipAllowed := len(addrList) > 0
-	var ipBlocked bool
-	for _, addr := range addrList {
-		ipAllowed = ipAllowed && allowList.MatchIPAddr(addr)
-		ipBlocked = ipBlocked || blockList.MatchIPAddr(addr)
-	}
-	if blockList.MatchHostName(hostName) || ipBlocked {
-		return &git.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true}
-	}
-	// if we have an allow-list, check the allow-list before return to get the more accurate error
-	if !allowList.IsEmpty() {
-		if !allowList.MatchHostName(hostName) && !ipAllowed {
-			return &git.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true}
-		}
-	}
-	// otherwise, we always follow the blocked list
 	return nil
 }
 
@@ -509,29 +479,3 @@ func migrateRepository(ctx context.Context, doer *user_model.User, downloader ba
 
 	return uploader.Finish(ctx)
 }
-
-// Init migrations service
-func Init() error {
-	// TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead
-
-	blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains)
-
-	allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains)
-	if allowList.IsEmpty() {
-		// the default policy is that migration module can access external hosts
-		allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal)
-	}
-	if setting.Migrations.AllowLocalNetworks {
-		allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate)
-		allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback)
-	} else {
-		blockList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate)
-		blockList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback)
-	}
-
-	// reset the shared client so it is rebuilt from the freshly parsed lists on next use; download paths
-	// then reuse one connection pool instead of creating a client (and pool) per request
-	migrationHTTPClient.Reset()
-
-	return nil
-}

services/migrations/migrate_test.go

@@ -6,15 +6,14 @@ package migrations
 import (
 	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"path/filepath"
 	"testing"
 
-	"gitea.dev/models/unittest"
 	user_model "gitea.dev/models/user"
 	"gitea.dev/modules/git/gitcmd"
 	"gitea.dev/modules/setting"
+	"gitea.dev/modules/test"
 	"gitea.dev/modules/util"
 
 	"github.com/google/go-github/v92/github"
@@ -43,102 +42,29 @@ func TestIsAuthenticationError(t *testing.T) {
 }
 
 func TestMigrateWhiteBlocklist(t *testing.T) {
-	assert.NoError(t, unittest.PrepareTestDatabase())
+	adminUser := &user_model.User{IsAdmin: true}
+	nonAdminUser := &user_model.User{}
 
-	adminUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user1"})
-	nonAdminUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{Name: "user2"})
-
-	setting.Migrations.AllowedDomains = "github.com"
-	setting.Migrations.AllowLocalNetworks = false
-	assert.NoError(t, Init())
-
-	err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
-	assert.Error(t, err)
-
-	err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
-	assert.NoError(t, err)
-
-	err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
-	assert.NoError(t, err)
-
-	setting.Migrations.AllowedDomains = ""
-	setting.Migrations.BlockedDomains = "github.com"
-	assert.NoError(t, Init())
-
-	err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
-	assert.NoError(t, err)
-
-	err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
-	assert.Error(t, err)
-
-	err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
-	assert.Error(t, err)
-
-	setting.Migrations.AllowLocalNetworks = true
-	assert.NoError(t, Init())
-	err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
-	assert.NoError(t, err)
+	defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")()
+	defer test.MockVariableValue(&setting.Migrations.BlockedHostList, "8.8.4.4")()
+	assert.NoError(t, IsMigrateURLAllowed("https://8.8.8.8/go-gitea/gitea.git", nonAdminUser))
+	assert.Error(t, IsMigrateURLAllowed("https://8.8.4.4/go-gitea/gitea.git", nonAdminUser))
+	assert.Error(t, IsMigrateURLAllowed("https://[64:ff9b::a9fe:a9fe]/go-gitea/gitea.git", nonAdminUser))
 
 	old := setting.ImportLocalPaths
 	setting.ImportLocalPaths = false
 
-	err = IsMigrateURLAllowed("/home/foo/bar/goo", adminUser)
-	assert.Error(t, err)
+	assert.Error(t, IsMigrateURLAllowed("/home/foo/bar/goo", adminUser))
 
 	setting.ImportLocalPaths = true
 	abs, err := filepath.Abs(".")
 	assert.NoError(t, err)
 
-	err = IsMigrateURLAllowed(abs, adminUser)
-	assert.NoError(t, err)
-
-	err = IsMigrateURLAllowed(abs, nonAdminUser)
-	assert.Error(t, err)
+	assert.NoError(t, IsMigrateURLAllowed(abs, adminUser))
+	assert.Error(t, IsMigrateURLAllowed(abs, nonAdminUser))
 
 	nonAdminUser.AllowImportLocal = true
-	err = IsMigrateURLAllowed(abs, nonAdminUser)
-	assert.NoError(t, err)
+	assert.NoError(t, IsMigrateURLAllowed(abs, nonAdminUser))
 
 	setting.ImportLocalPaths = old
 }
-
-func TestAllowBlockList(t *testing.T) {
-	init := func(allow, block string, local bool) {
-		setting.Migrations.AllowedDomains = allow
-		setting.Migrations.BlockedDomains = block
-		setting.Migrations.AllowLocalNetworks = local
-		assert.NoError(t, Init())
-	}
-
-	// default, allow all external, block none, no local networks
-	init("", "", false)
-	assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")}))
-
-	// allow all including local networks (it could lead to SSRF in production)
-	init("", "", true)
-	assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")}))
-
-	// allow wildcard, block some subdomains. every resolved address must still be allowed.
-	init("*.domain.com", "blocked.domain.com", false)
-	assert.NoError(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.Error(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("127.0.0.1")}))
-	assert.Error(t, checkByAllowBlockList("sub.domain.com", []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("127.0.0.1")}))
-	assert.Error(t, checkByAllowBlockList("blocked.domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.Error(t, checkByAllowBlockList("sub.other.com", []net.IP{net.ParseIP("1.2.3.4")}))
-
-	// allow wildcard still follows the local network policy for resolved addresses.
-	init("*", "", false)
-	assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")}))
-	assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4"), net.ParseIP("127.0.0.1")}))
-
-	// local network can still be blocked
-	init("*", "127.0.0.*", false)
-	assert.NoError(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("1.2.3.4")}))
-	assert.Error(t, checkByAllowBlockList("domain.com", []net.IP{net.ParseIP("127.0.0.1")}))
-
-	// reset to allow local networks (mock servers use 127.0.0.1)
-	init("", "", true)
-}

services/migrations/onedev.go

@@ -88,10 +88,11 @@ func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, passwo
 		client: &http.Client{
 			Transport: roundTripperFunc(
 				func(req *http.Request) (*http.Response, error) {
+					req = req.Clone(ctx)
 					if username != "" && password != "" {
 						req.SetBasicAuth(username, password)
 					}
-					return httpTransport.RoundTrip(req.WithContext(ctx))
+					return httpTransport.RoundTrip(req)
 				}),
 		},
 		userMap:      make(map[int64]*onedevUser),

services/mirror/mirror_pull.go

@@ -18,7 +18,6 @@ import (
 	"gitea.dev/modules/lfs"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	repo_module "gitea.dev/modules/repository"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -69,9 +68,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
 }
 
 func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, repoLogName string, gitRepo git.RepositoryFacade, timeout time.Duration) error {
-	cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
-	git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
-	stdout, _, pruneErr := cmd.WithRepo(gitRepo).RunStdString(ctx)
+	stdout, _, pruneErr := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithRepo(gitRepo).RunStdString(ctx)
 	if pruneErr != nil {
 		// sanitize the output, since it may contain the remote address, which may contain a password
 		stderrMessage := util.SanitizeCredentialURLs(pruneErr.Stderr())
@@ -122,17 +119,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu
 			return nil, false
 		}
 	}
-	envs := proxy.EnvWithProxy(remoteURL.URL)
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
 
 	// use fetch but not remote update because git fetch support --tags but remote update doesn't
 	cmdFetch := func() *gitcmd.Command {
 		cmd := gitcmd.NewCommand("fetch", "--tags")
-		git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
 		if m.EnablePrune {
 			cmd.AddArguments("--prune")
 		}
-		return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs)
+		return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
 	}
 
 	var err error
@@ -209,9 +204,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu
 	}
 
 	cmdRemoteUpdatePrune := func() *gitcmd.Command {
-		cmd := gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs)
-		git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
-		return cmd
+		return gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {

services/mirror/mirror_push.go

@@ -18,7 +18,6 @@ import (
 	"gitea.dev/modules/lfs"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/repository"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -156,13 +155,11 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error {
 
 		log.Trace("Pushing mirror %d repo %s to remote %s", m.ID, storageRepo.LogString(), m.RemoteName)
 
-		envs := proxy.EnvWithProxy(remoteURL.URL)
 		if err := git.PushToExternal(ctx, storageRepo, git.PushOptions{
 			Remote:  m.RemoteName,
 			Force:   true,
 			Mirror:  true,
 			Timeout: timeout,
-			Env:     envs,
 		}); err != nil {
 			return fmt.Errorf("PushToExternal failed: %w", err)
 		}

services/webhook/deliver.go

@@ -16,17 +16,14 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
 	"time"
 
 	user_model "gitea.dev/models/user"
 	webhook_model "gitea.dev/models/webhook"
-	"gitea.dev/modules/glob"
+	"gitea.dev/modules/egress"
 	"gitea.dev/modules/graceful"
-	"gitea.dev/modules/hostmatcher"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/queue"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -271,52 +268,15 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
 	return nil
 }
 
-var (
-	webhookHTTPClient *http.Client
-	once              sync.Once
-	hostMatchers      []glob.Glob
-)
-
-func webhookProxy(allowList *hostmatcher.HostMatchList) func(req *http.Request) (*url.URL, error) {
-	if setting.Webhook.ProxyURL == "" {
-		return proxy.Proxy()
-	}
-
-	once.Do(func() {
-		for _, h := range setting.Webhook.ProxyHosts {
-			if g, err := glob.Compile(h); err == nil {
-				hostMatchers = append(hostMatchers, g)
-			} else {
-				log.Error("glob.Compile %s failed: %v", h, err)
-			}
-		}
-	})
-
-	return func(req *http.Request) (*url.URL, error) {
-		for _, v := range hostMatchers {
-			if v.Match(req.URL.Host) {
-				if !allowList.MatchHostName(req.URL.Host) {
-					return nil, fmt.Errorf("webhook can only call allowed HTTP servers (check your %s setting), deny '%s'", allowList.SettingKeyHint, req.URL.Host)
-				}
-				return http.ProxyURL(setting.Webhook.ProxyURLFixed)(req)
-			}
-		}
-		return http.ProxyFromEnvironment(req)
-	}
-}
+var webhookHTTPClient *http.Client
 
 // Init starts the hooks delivery thread
 func Init() error {
 	timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
-	allowedHostMatcher := hostmatcher.ParseHostMatchList("security.ALLOWED_HOST_LIST", setting.Webhook.AllowedHostList)
 
-	// NewHTTPTransport enforces the allow-list on direct connections; when webhookProxy routes a request
-	// through a configured proxy, restricting the proxied target is the proxy server's responsibility.
-	webhookHTTPClient = &http.Client{
-		Timeout: timeout,
-		Transport: hostmatcher.NewHTTPTransport("webhook", allowedHostMatcher, nil, webhookProxy(allowedHostMatcher), setting.Webhook.ProxyURLFixed,
-			&tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}),
-	}
+	transport := egress.NewWebhookPolicy().NewHTTPTransport()
+	transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify}
+	webhookHTTPClient = &http.Client{Timeout: timeout, Transport: transport}
 
 	hookQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "webhook_sender", handler)
 	if hookQueue == nil {

services/webhook/deliver_test.go

@@ -7,82 +7,18 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"net/url"
 	"strings"
 	"testing"
 	"time"
 
 	"gitea.dev/models/unittest"
 	webhook_model "gitea.dev/models/webhook"
-	"gitea.dev/modules/hostmatcher"
-	"gitea.dev/modules/setting"
 	"gitea.dev/modules/util"
 	webhook_module "gitea.dev/modules/webhook"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-func TestWebhookProxy(t *testing.T) {
-	oldWebhook := setting.Webhook
-	t.Cleanup(func() {
-		setting.Webhook = oldWebhook
-	})
-
-	setting.Webhook.ProxyURL = "http://localhost:8080"
-	setting.Webhook.ProxyURLFixed, _ = url.Parse(setting.Webhook.ProxyURL)
-	setting.Webhook.ProxyHosts = []string{"*.discordapp.com", "discordapp.com"}
-
-	allowedHostMatcher := hostmatcher.ParseHostMatchList("webhook.ALLOWED_HOST_LIST", "discordapp.com,s.discordapp.com")
-
-	tests := []struct {
-		req     string
-		want    string
-		wantErr bool
-	}{
-		{
-			req:     "https://discordapp.com/api/webhooks/xxxxxxxxx/xxxxxxxxxxxxxxxxxxx",
-			want:    "http://localhost:8080",
-			wantErr: false,
-		},
-		{
-			req:     "http://s.discordapp.com/assets/xxxxxx",
-			want:    "http://localhost:8080",
-			wantErr: false,
-		},
-		{
-			req:     "http://github.com/a/b",
-			want:    "",
-			wantErr: false,
-		},
-		{
-			req:     "http://www.discordapp.com/assets/xxxxxx",
-			want:    "",
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.req, func(t *testing.T) {
-			req, err := http.NewRequest(http.MethodPost, tt.req, nil)
-			require.NoError(t, err)
-
-			u, err := webhookProxy(allowedHostMatcher)(req)
-			if tt.wantErr {
-				assert.Error(t, err)
-				return
-			}
-
-			assert.NoError(t, err)
-
-			got := ""
-			if u != nil {
-				got = u.String()
-			}
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
 func TestWebhookDeliverAuthorizationHeader(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 

services/webhook/main_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"gitea.dev/models/unittest"
-	"gitea.dev/modules/hostmatcher"
 	"gitea.dev/modules/setting"
 
 	_ "gitea.dev/models"
@@ -17,9 +16,7 @@ import (
 func TestMain(m *testing.M) {
 	unittest.MainTest(m, &unittest.TestOptions{
 		SetUp: func() error {
-			// for tests, allow only loopback IPs. This must run after the test config is loaded (which
-			// resets the shared Security.AllowedHostList) and before Init() builds the delivery client.
-			setting.Security.AllowedHostList = hostmatcher.MatchBuiltinLoopback
+			setting.Webhook.AllowedHostList = "loopback:*"
 			setting.LoadQueueSettings()
 			return Init()
 		},