fix(git)!: use internal proxy for all git operations (#39426)

Introduces gitproxy module which spawns a small forward proxy as scanner
for git calls
Replaces hostmatcher with matchlist which supports port rules
Deprecates ALLOWED_DOMAINS/BLOCKED_DOMAINS and ALLOW_LOCALNETWORKS
settings in migration in favor of full names we have in security
configs.
Removes `external` preset in favor of lax/strict modes, strict mode
requiring explicit ports if they aren't standard http/s ones.

Breaking changes:
- `external` preset no longer works as deny rule. To enforce that, use
`strict` mode and allow ranges to connect to
- Wildcards are no longer accepted in IP addresses
- `*` is no longer allowed as entry in lists
- domain rules now use curl like syntax `*.example.com` matching
subdomains but not `example.com`, `example.com` matching itself and all
subdomains. `example.*` is not a valid rule
- In the default `lax` mode, `[security] ALLOWED_HOST_LIST` no longer
restricts public hosts, set `EGRESS_MODE = strict` to keep an exclusive
list. A startup warning flags this
- Invalid list entries are logged at startup, invalid
`BLOCKED_HOST_LIST`/`BLOCKED_DOMAINS` entries stop it

Docs: https://gitea.com/gitea/docs/pulls/557
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

tests/integration/mirror_pull_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"slices"
+	"strings"
 	"sync/atomic"
 	"testing"
 
@@ -20,7 +21,6 @@ import (
 	"gitea.dev/modules/migration"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/test"
-	migrations "gitea.dev/services/migrations"
 	mirror_service "gitea.dev/services/mirror"
 	release_service "gitea.dev/services/release"
 	repo_service "gitea.dev/services/repository"
@@ -139,40 +139,19 @@ func TestMirrorPullSSRFRevalidation(t *testing.T) {
 	defer tests.PrepareTestEnv(t)()
 
 	ctx := t.Context()
-	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
-	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
-	repoPath := gitrepo.RepoLocalPath(repo)
-
-	// an "internal" server that records whether it was reached
-	var reached atomic.Bool
-	internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		reached.Store(true)
-		w.WriteHeader(http.StatusNotFound)
+	var requests atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		http.Redirect(w, r, "http://"+strings.Replace(r.Host, "127.0.0.1", "0.0.0.0", 1)+r.URL.RequestURI(), http.StatusFound)
 	}))
-	defer internal.Close()
+	defer server.Close()
 
-	mirrorRepo, err := repo_service.CreateRepositoryDirectly(ctx, user, user, repo_service.CreateRepoOptions{
-		Name:     "ssrf_mirror",
-		IsMirror: true,
-		Status:   repo_model.RepositoryBeingMigrated,
-	}, false)
-	require.NoError(t, err)
-	_, err = repo_service.MigrateRepositoryGitData(ctx, user, mirrorRepo, migration.MigrateOptions{
-		RepoName:  "ssrf_mirror",
-		Mirror:    true,
-		CloneAddr: repoPath,
-	}, nil)
-	require.NoError(t, err)
+	mirror := unittest.AssertExistsAndLoadBean(t, &repo_model.Mirror{RepoID: 5})
+	require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, server.URL+"/repo.git"))
+	assert.False(t, mirror_service.SyncPullMirror(ctx, mirror.RepoID))
+	assert.EqualValues(t, 1, requests.Load(), "the git proxy must deny the redirect to a reserved address")
 
-	mirror, err := repo_model.GetMirrorByRepoID(ctx, mirrorRepo.ID)
-	require.NoError(t, err)
-
-	// repoint the mirror at the loopback server, which is disallowed once local networks are off
-	require.NoError(t, mirror_service.UpdateAddress(ctx, mirror, internal.URL+"/repo.git"))
-	defer test.MockVariableValue(&setting.Migrations.AllowLocalNetworks, false)()
-	require.NoError(t, migrations.Init())
-	t.Cleanup(func() { _ = migrations.Init() })
-
-	assert.False(t, mirror_service.SyncPullMirror(ctx, mirrorRepo.ID))
-	assert.False(t, reached.Load(), "the disallowed internal remote must not be reached")
+	defer test.MockVariableValue(&setting.Migrations.AllowedHostList, "")()
+	assert.False(t, mirror_service.SyncPullMirror(ctx, mirror.RepoID))
+	assert.EqualValues(t, 1, requests.Load(), "the disallowed internal remote must not be reached")
 }