Changelog for v1.15.7 (#17871)

* Changelog for v1.15.7

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: 6543 <6543@obermui.de>

.golangci.yml

@@ -9,7 +9,6 @@ linters:
     - unused
     - structcheck
     - varcheck
-    - golint
     - dupl
     #- gocyclo # The cyclomatic complexety of a lot of functions is too high, we should refactor those another time.
     - gofmt

CHANGELOG.md

@@ -4,6 +4,65 @@ This changelog goes through all the changes that have been made in each release
 without substantial changes to our git log; to see the highlights of what has
 been added to each release, please refer to the [blog](https://blog.gitea.io).
 
+## [1.15.7](https://github.com/go-gitea/gitea/releases/tag/v1.15.7) - 2021-12-01
+
+* ENHANCEMENTS
+  * Only allow webhook to send requests to allowed hosts (#17482) (#17510)
+  * Fix login redirection links (#17451) (#17473)
+* BUGFIXES
+  * Fix database inconsistent when admin change user email (#17549) (#17840)
+  * Use correct user on releases (#17806) (#17818)
+  * Fix commit count in tag view (#17698) (#17790)
+  * Fix close issue but time watcher still running (#17643) (#17761)
+  * Fix Migrate Description (#17692) (#17727)
+  * Fix bug when project board get open issue number (#17703) (#17726)
+  * Return 400 but not 500 when request archive with wrong format (#17691) (#17700)
+  * Fix bug when read mysql database max lifetime (#17682) (#17690)
+  * Fix database deadlock when update issue labels (#17649) (#17665)
+  * Fix bug on detect issue/comment writer (#17592)
+  * Remove appSubUrl from pasted images (#17572) (#17588)
+  * Make `ParsePatch` more robust (#17573) (#17580)
+  * Fix stats upon searching issues (#17566) (#17578)
+  * Escape issue titles in comments list (#17555) (#17556)
+  * Fix zero created time bug on commit api (#17546) (#17547)
+  * Fix database keyword quote problem on migration v161 (#17522) (#17523)
+  * Fix email with + when active (#17518) (#17520)
+  * Stop double encoding blame commit messages (#17498) (#17500)
+  * Quote the table name in CountOrphanedObjects (#17487) (#17488)
+  * Run Migrate in Install rather than just SyncTables (#17475) (#17486)
+* BUILD
+  * Fix golangci-lint warnings (#17598 et al) (#17668)
+* MISC
+  * Preserve color when inverting emojis (#17797) (#17799)
+
+## [1.15.6](https://github.com/go-gitea/gitea/releases/tag/v1.15.6) - 2021-10-28
+
+* BUGFIXES
+  * Prevent panic in serv.go with Deploy Keys (#17434) (#17435)
+  * Fix CSV render error (#17406) (#17431)
+  * Read expected buffer size (#17409) (#17430)
+  * Ensure that restricted users can access repos for which they are members (#17460) (#17464)
+  * Make commit-statuses popup show correctly (#17447) (#17466)
+* TESTING
+  * Add integration tests for private.NoServCommand and private.ServCommand (#17456) (#17463)
+
+## [1.15.5](https://github.com/go-gitea/gitea/releases/tag/v1.15.5) - 2021-10-21
+
+* SECURITY
+  * Upgrade Bluemonday to v1.0.16 (#17372) (#17374)
+  * Ensure correct SSH permissions check for private and restricted users (#17370) (#17373)
+* BUGFIXES
+  * Prevent NPE in CSV diff rendering when column removed (#17018) (#17377)
+  * Offer rsa-sha2-512 and rsa-sha2-256 algorithms in internal SSH (#17281) (#17376)
+  * Don't panic if we fail to parse U2FRegistration data (#17304) (#17371)
+  * Ensure popup text is aligned left (backport for 1.15) (#17343)
+  * Ensure that git daemon export ok is created for mirrors (#17243) (#17306)
+  * Disable core.protectNTFS (#17300) (#17302)
+  * Use pointer for wrappedConn methods (#17295) (#17296)
+  * AutoRegistration is supposed to be working with disabled registration (backport) (#17292)
+  * Handle duplicate keys on GPG key ring (#17242) (#17284)
+  * Fix SVG side by side comparison link (#17375) (#17391)
+
 ## [1.15.4](https://github.com/go-gitea/gitea/releases/tag/v1.15.4) - 2021-10-08
 * BUGFIXES
   * Raw file API: don't try to interpret 40char filenames as commit SHA (#17185) (#17272)

build.go

@@ -2,7 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-//+build vendor
+//go:build vendor
+// +build vendor
 
 package main
 

build/generate-bindata.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build ignore
 // +build ignore
 
 package main

build/generate-emoji.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build ignore
 // +build ignore
 
 package main

build/generate-gitignores.go

@@ -1,3 +1,4 @@
+//go:build ignore
 // +build ignore
 
 package main

build/generate-licenses.go

@@ -1,3 +1,4 @@
+//go:build ignore
 // +build ignore
 
 package main

build/gocovmerge.go

@@ -6,6 +6,7 @@
 // gocovmerge takes the results from multiple `go test -coverprofile` runs and
 // merges them into one profile
 
+//go:build ignore
 // +build ignore
 
 package main

cmd/docs.go

@@ -43,7 +43,11 @@ func runDocs(ctx *cli.Context) error {
 		// Clean up markdown. The following bug was fixed in v2, but is present in v1.
 		// It affects markdown output (even though the issue is referring to man pages)
 		// https://github.com/urfave/cli/issues/1040
-		docs = docs[strings.Index(docs, "#"):]
+		firstHashtagIndex := strings.Index(docs, "#")
+
+		if firstHashtagIndex > 0 {
+			docs = docs[firstHashtagIndex:]
+		}
 	}
 
 	out := os.Stdout

cmd/embedded.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build bindata
 // +build bindata
 
 package cmd

cmd/embedded_stub.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !bindata
 // +build !bindata
 
 package cmd

cmd/web.go

@@ -194,6 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error {
 		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
 	}
 	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)
+	// This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy.
+	// A user may fix the configuration mistake when he sees this log.
+	// And this is also very helpful to maintainers to provide help to users to resolve their configuration problems.
+	log.Info("AppURL(ROOT_URL): %s", setting.AppURL)
 
 	if setting.LFS.StartServer {
 		log.Info("LFS server enabled")

custom/conf/app.example.ini

@@ -576,6 +576,8 @@ PATH =
 ;;
 ;; (Go-Git only) Don't cache objects greater than this in memory. (Set to 0 to disable.)
 ;LARGE_OBJECT_THRESHOLD = 1048576
+;; Set to true to forcibly set core.protectNTFS=false
+;DISABLE_CORE_PROTECT_NTFS=false
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1386,6 +1388,13 @@ PATH =
 ;; Deliver timeout in seconds
 ;DELIVER_TIMEOUT = 5
 ;;
+;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com
+;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts)
+;; CIDR list: 1.2.3.0/8, 2001:db8::/32
+;; Wildcard hosts: *.mydomain.com, 192.168.100.*
+;; Default to * for 1.15.x, external for 1.16 and later
+;ALLOWED_HOST_LIST = *
+;;
 ;; Allow insecure certification
 ;SKIP_TLS_VERIFY = false
 ;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -545,6 +545,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
 
 - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value.
 - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
+- `ALLOWED_HOST_LIST`: `*`: Default to `*` for 1.15.x, `external` for 1.16 and later. Webhook can only call allowed hosts for security reasons. Comma separated list.
+  - Built-in networks:
+    - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+    - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+    - `external`: A valid non-private unicast IP, you can access all hosts on public internet. 
+    - `*`: All hosts are allowed.
+  - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6
+  - Wildcard hosts: `*.mydomain.com`, `192.168.100.*`
 - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
 - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
 - `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
@@ -839,6 +847,7 @@ NB: You must have `DISABLE_ROUTER_LOG` set to `false` for this option to take ef
 - `VERBOSE_PUSH`: **true**: Print status information about pushes as they are being processed.
 - `VERBOSE_PUSH_DELAY`: **5s**: Only print verbose information if push takes longer than this delay.
 - `LARGE_OBJECT_THRESHOLD`: **1048576**: (Go-Git only), don't cache objects greater than this in memory. (Set to 0 to disable.)
+- `DISABLE_CORE_PROTECT_NTFS`: **false** Set to true to forcibly set `core.protectNTFS` to false.
 ## Git - Timeout settings (`git.timeout`)
 - `DEFAUlT`: **360**: Git operations default timeout seconds.
 - `MIGRATE`: **600**: Migrate external repositories timeout seconds.

go.mod

@@ -80,7 +80,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/mholt/archiver/v3 v3.5.0
-	github.com/microcosm-cc/bluemonday v1.0.15
+	github.com/microcosm-cc/bluemonday v1.0.16
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/minio-go/v7 v7.0.12
@@ -125,7 +125,7 @@ require (
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.18.1 // indirect
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/net v0.0.0-20211020060615-d418f374d309
 	golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 	golang.org/x/text v0.3.6

go.sum

@@ -868,8 +868,8 @@ github.com/mholt/acmez v0.1.3 h1:J7MmNIk4Qf9b8mAGqAh4XkNeowv3f1zW816yf4zt7Qk=
 github.com/mholt/acmez v0.1.3/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM=
 github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=
 github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc=
-github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY=
-github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30=
+github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
+github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
@@ -1364,8 +1364,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/net v0.0.0-20210331060903-cb1fcc7394e5/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
+golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

integrations/api_private_serv_test.go

@@ -0,0 +1,154 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/private"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIPrivateNoServ(t *testing.T) {
+	onGiteaRun(t, func(*testing.T, *url.URL) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		key, user, err := private.ServNoCommand(ctx, 1)
+		assert.NoError(t, err)
+		assert.Equal(t, int64(2), user.ID)
+		assert.Equal(t, "user2", user.Name)
+		assert.Equal(t, int64(1), key.ID)
+		assert.Equal(t, "user2@localhost", key.Name)
+
+		deployKey, err := models.AddDeployKey(1, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", false)
+		assert.NoError(t, err)
+
+		key, user, err = private.ServNoCommand(ctx, deployKey.KeyID)
+		assert.NoError(t, err)
+		assert.Empty(t, user)
+		assert.Equal(t, deployKey.KeyID, key.ID)
+		assert.Equal(t, "test-deploy", key.Name)
+	})
+}
+
+func TestAPIPrivateServ(t *testing.T) {
+	onGiteaRun(t, func(*testing.T, *url.URL) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		// Can push to a repo we own
+		results, err := private.ServCommand(ctx, 1, "user2", "repo1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.False(t, results.IsDeployKey)
+		assert.Equal(t, int64(1), results.KeyID)
+		assert.Equal(t, "user2@localhost", results.KeyName)
+		assert.Equal(t, "user2", results.UserName)
+		assert.Equal(t, int64(2), results.UserID)
+		assert.Equal(t, "user2", results.OwnerName)
+		assert.Equal(t, "repo1", results.RepoName)
+		assert.Equal(t, int64(1), results.RepoID)
+
+		// Cannot push to a private repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a private repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_private_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Can pull from a public repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_public_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.False(t, results.IsDeployKey)
+		assert.Equal(t, int64(1), results.KeyID)
+		assert.Equal(t, "user2@localhost", results.KeyName)
+		assert.Equal(t, "user2", results.UserName)
+		assert.Equal(t, int64(2), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_public_1", results.RepoName)
+		assert.Equal(t, int64(17), results.RepoID)
+
+		// Cannot push to a public repo we're not associated with
+		results, err = private.ServCommand(ctx, 1, "user15", "big_test_public_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Add reading deploy key
+		deployKey, err := models.AddDeployKey(19, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", true)
+		assert.NoError(t, err)
+
+		// Can pull from repo we're a deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_1", results.RepoName)
+		assert.Equal(t, int64(19), results.RepoID)
+
+		// Cannot push to a private repo with reading key
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a private repo we're not associated with
+		results, err = private.ServCommand(ctx, deployKey.ID, "user15", "big_test_private_2", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Cannot pull from a public repo we're not associated with
+		results, err = private.ServCommand(ctx, deployKey.ID, "user15", "big_test_public_1", models.AccessModeRead, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Add writing deploy key
+		deployKey, err = models.AddDeployKey(20, "test-deploy", "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBGXEEzWmm1dxb+57RoK5KVCL0w2eNv9cqJX2AGGVlkFsVDhOXHzsadS3LTK4VlEbbrDMJdoti9yM8vclA8IeRacAAAAEc3NoOg== nocomment", false)
+		assert.NoError(t, err)
+
+		// Cannot push to a private repo with reading key
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_1", models.AccessModeWrite, "git-upload-pack", "")
+		assert.Error(t, err)
+		assert.Empty(t, results)
+
+		// Can pull from repo we're a writing deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_2", models.AccessModeRead, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_2", results.RepoName)
+		assert.Equal(t, int64(20), results.RepoID)
+
+		// Can push to repo we're a writing deploy key for
+		results, err = private.ServCommand(ctx, deployKey.KeyID, "user15", "big_test_private_2", models.AccessModeWrite, "git-upload-pack", "")
+		assert.NoError(t, err)
+		assert.False(t, results.IsWiki)
+		assert.True(t, results.IsDeployKey)
+		assert.Equal(t, deployKey.KeyID, results.KeyID)
+		assert.Equal(t, "test-deploy", results.KeyName)
+		assert.Equal(t, "user15", results.UserName)
+		assert.Equal(t, int64(15), results.UserID)
+		assert.Equal(t, "user15", results.OwnerName)
+		assert.Equal(t, "big_test_private_2", results.RepoName)
+		assert.Equal(t, int64(20), results.RepoID)
+
+	})
+
+}

integrations/api_repo_archive_test.go

@@ -0,0 +1,44 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPIDownloadArchive(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
+	user2 := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User)
+	session := loginUser(t, user2.LowerName)
+	token := getTokenForLoggedInUser(t, session)
+
+	link, _ := url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master.zip", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	resp := MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	bs, err := io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 320, len(bs))
+
+	link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master.tar.gz", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	resp = MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	bs, err = io.ReadAll(resp.Body)
+	assert.NoError(t, err)
+	assert.EqualValues(t, 266, len(bs))
+
+	link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", user2.Name, repo.Name))
+	link.RawQuery = url.Values{"token": {token}}.Encode()
+	MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusBadRequest)
+}

integrations/api_user_heatmap_test.go

@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"net/http"
 	"testing"
+	"time"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/timeutil"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -20,6 +22,10 @@ func TestUserHeatmap(t *testing.T) {
 	normalUsername := "user2"
 	session := loginUser(t, adminUsername)
 
+	var fakeNow = time.Date(2011, 10, 20, 0, 0, 0, 0, time.Local)
+	timeutil.Set(fakeNow)
+	defer timeutil.Unset()
+
 	urlStr := fmt.Sprintf("/api/v1/users/%s/heatmap", normalUsername)
 	req := NewRequest(t, "GET", urlStr)
 	resp := session.MakeRequest(t, req, http.StatusOK)

integrations/org_test.go

@@ -5,10 +5,12 @@
 package integrations
 
 import (
+	"fmt"
 	"net/http"
 	"strings"
 	"testing"
 
+	api "code.gitea.io/gitea/modules/structs"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -110,3 +112,64 @@ func TestPrivateOrg(t *testing.T) {
 	req = NewRequest(t, "GET", "/privated_org/private_repo_on_private_org")
 	session.MakeRequest(t, req, http.StatusOK)
 }
+
+func TestOrgRestrictedUser(t *testing.T) {
+	defer prepareTestEnv(t)()
+
+	// privated_org is a private org who has id 23
+	orgName := "privated_org"
+
+	// public_repo_on_private_org is a public repo on privated_org
+	repoName := "public_repo_on_private_org"
+
+	// user29 is a restricted user who is not a member of the organization
+	restrictedUser := "user29"
+
+	// #17003 reports a bug whereby adding a restricted user to a read-only team doesn't work
+
+	// assert restrictedUser cannot see the org or the public repo
+	restrictedSession := loginUser(t, restrictedUser)
+	req := NewRequest(t, "GET", fmt.Sprintf("/%s", orgName))
+	restrictedSession.MakeRequest(t, req, http.StatusNotFound)
+
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s/%s", orgName, repoName))
+	restrictedSession.MakeRequest(t, req, http.StatusNotFound)
+
+	// Therefore create a read-only team
+	adminSession := loginUser(t, "user1")
+	token := getTokenForLoggedInUser(t, adminSession)
+
+	teamToCreate := &api.CreateTeamOption{
+		Name:                    "codereader",
+		Description:             "Code Reader",
+		IncludesAllRepositories: true,
+		Permission:              "read",
+		Units:                   []string{"repo.code"},
+	}
+
+	req = NewRequestWithJSON(t, "POST",
+		fmt.Sprintf("/api/v1/orgs/%s/teams?token=%s", orgName, token), teamToCreate)
+
+	var apiTeam api.Team
+
+	resp := adminSession.MakeRequest(t, req, http.StatusCreated)
+	DecodeJSON(t, resp, &apiTeam)
+	checkTeamResponse(t, &apiTeam, teamToCreate.Name, teamToCreate.Description, teamToCreate.IncludesAllRepositories,
+		teamToCreate.Permission, teamToCreate.Units)
+	checkTeamBean(t, apiTeam.ID, teamToCreate.Name, teamToCreate.Description, teamToCreate.IncludesAllRepositories,
+		teamToCreate.Permission, teamToCreate.Units)
+	//teamID := apiTeam.ID
+
+	// Now we need to add the restricted user to the team
+	req = NewRequest(t, "PUT",
+		fmt.Sprintf("/api/v1/teams/%d/members/%s?token=%s", apiTeam.ID, restrictedUser, token))
+	_ = adminSession.MakeRequest(t, req, http.StatusNoContent)
+
+	// Now we need to check if the restrictedUser can access the repo
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s", orgName))
+	restrictedSession.MakeRequest(t, req, http.StatusOK)
+
+	req = NewRequest(t, "GET", fmt.Sprintf("/%s/%s", orgName, repoName))
+	restrictedSession.MakeRequest(t, req, http.StatusOK)
+
+}

models/consistency.go

@@ -302,7 +302,7 @@ func DeleteOrphanedIssues() error {
 // CountOrphanedObjects count subjects with have no existing refobject anymore
 func CountOrphanedObjects(subject, refobject, joinCond string) (int64, error) {
 	return x.Table("`"+subject+"`").
-		Join("LEFT", refobject, joinCond).
+		Join("LEFT", "`"+refobject+"`", joinCond).
 		Where(builder.IsNull{"`" + refobject + "`.id"}).
 		Count("id")
 }

models/fixtures/repository.yml

@@ -568,7 +568,7 @@
 -
   id: 40
   owner_id: 23
-  owner_name: limited_org
+  owner_name: privated_org
   lower_name: public_repo_on_private_org
   name: public_repo_on_private_org
   is_private: false
@@ -581,7 +581,7 @@
 -
   id: 41
   owner_id: 23
-  owner_name: limited_org
+  owner_name: privated_org
   lower_name: private_repo_on_private_org
   name: private_repo_on_private_org
   is_private: true

models/gpg_key_add.go

@@ -99,6 +99,46 @@ func AddGPGKey(ownerID int64, content, token, signature string) ([]*GPGKey, erro
 		verified = true
 	}
 
+	if len(ekeys) > 1 {
+		id2key := map[string]*openpgp.Entity{}
+		newEKeys := make([]*openpgp.Entity, 0, len(ekeys))
+		for _, ekey := range ekeys {
+			id := ekey.PrimaryKey.KeyIdString()
+			if original, has := id2key[id]; has {
+				// Coalesce this with the other one
+				for _, subkey := range ekey.Subkeys {
+					if subkey.PublicKey == nil {
+						continue
+					}
+					found := false
+
+					for _, originalSubkey := range original.Subkeys {
+						if originalSubkey.PublicKey == nil {
+							continue
+						}
+						if originalSubkey.PublicKey.KeyId == subkey.PublicKey.KeyId {
+							found = true
+							break
+						}
+					}
+					if !found {
+						original.Subkeys = append(original.Subkeys, subkey)
+					}
+				}
+				for name, identity := range ekey.Identities {
+					if _, has := original.Identities[name]; has {
+						continue
+					}
+					original.Identities[name] = identity
+				}
+				continue
+			}
+			id2key[id] = ekey
+			newEKeys = append(newEKeys, ekey)
+		}
+		ekeys = newEKeys
+	}
+
 	for _, ekey := range ekeys {
 		// Key ID cannot be duplicated.
 		has, err := sess.Where("key_id=?", ekey.PrimaryKey.KeyIdString()).

models/issue.go

@@ -1517,12 +1517,12 @@ func GetIssueStats(opts *IssueStatsOptions) (*IssueStats, error) {
 func getIssueStatsChunk(opts *IssueStatsOptions, issueIDs []int64) (*IssueStats, error) {
 	stats := &IssueStats{}
 
-	countSession := func(opts *IssueStatsOptions) *xorm.Session {
+	countSession := func(opts *IssueStatsOptions, issueIDs []int64) *xorm.Session {
 		sess := x.
 			Where("issue.repo_id = ?", opts.RepoID)
 
-		if len(opts.IssueIDs) > 0 {
-			sess.In("issue.id", opts.IssueIDs)
+		if len(issueIDs) > 0 {
+			sess.In("issue.id", issueIDs)
 		}
 
 		if len(opts.Labels) > 0 && opts.Labels != "0" {
@@ -1572,13 +1572,13 @@ func getIssueStatsChunk(opts *IssueStatsOptions, issueIDs []int64) (*IssueStats,
 	}
 
 	var err error
-	stats.OpenCount, err = countSession(opts).
+	stats.OpenCount, err = countSession(opts, issueIDs).
 		And("issue.is_closed = ?", false).
 		Count(new(Issue))
 	if err != nil {
 		return stats, err
 	}
-	stats.ClosedCount, err = countSession(opts).
+	stats.ClosedCount, err = countSession(opts, issueIDs).
 		And("issue.is_closed = ?", true).
 		Count(new(Issue))
 	return stats, err

models/issue_stopwatch.go

@@ -13,6 +13,26 @@ import (
 	"xorm.io/xorm"
 )
 
+// ErrIssueStopwatchNotExist represents an error that stopwatch is not exist
+type ErrIssueStopwatchNotExist struct {
+	UserID  int64
+	IssueID int64
+}
+
+func (err ErrIssueStopwatchNotExist) Error() string {
+	return fmt.Sprintf("issue stopwatch doesn't exist[uid: %d, issue_id: %d", err.UserID, err.IssueID)
+}
+
+// ErrIssueStopwatchAlreadyExist represents an error that stopwatch is already exist
+type ErrIssueStopwatchAlreadyExist struct {
+	UserID  int64
+	IssueID int64
+}
+
+func (err ErrIssueStopwatchAlreadyExist) Error() string {
+	return fmt.Sprintf("issue stopwatch already exists[uid: %d, issue_id: %d", err.UserID, err.IssueID)
+}
+
 // Stopwatch represents a stopwatch for time tracking.
 type Stopwatch struct {
 	ID          int64              `xorm:"pk autoincr"`
@@ -74,91 +94,141 @@ func hasUserStopwatch(e Engine, userID int64) (exists bool, sw *Stopwatch, err e
 	return
 }
 
+// FinishIssueStopwatchIfPossible if stopwatch exist then finish it otherwise ignore
+func FinishIssueStopwatchIfPossible(user *User, issue *Issue) error {
+	_, exists, err := getStopwatch(x, user.ID, issue.ID)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+	return FinishIssueStopwatch(user, issue)
+}
+
 // CreateOrStopIssueStopwatch will create or remove a stopwatch and will log it into issue's timeline.
 func CreateOrStopIssueStopwatch(user *User, issue *Issue) error {
+	_, exists, err := getStopwatch(x, user.ID, issue.ID)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return FinishIssueStopwatch(user, issue)
+	}
+	return CreateIssueStopwatch(user, issue)
+}
+
+// FinishIssueStopwatch if stopwatch exist then finish it otherwise return an error
+func FinishIssueStopwatch(user *User, issue *Issue) error {
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
 		return err
 	}
-	if err := createOrStopIssueStopwatch(sess, user, issue); err != nil {
+	if err := finishIssueStopwatch(sess, user, issue); err != nil {
 		return err
 	}
 	return sess.Commit()
 }
 
-func createOrStopIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
+func finishIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
 	sw, exists, err := getStopwatch(e, user.ID, issue.ID)
 	if err != nil {
 		return err
 	}
+	if !exists {
+		return ErrIssueStopwatchNotExist{
+			UserID:  user.ID,
+			IssueID: issue.ID,
+		}
+	}
+
+	// Create tracked time out of the time difference between start date and actual date
+	timediff := time.Now().Unix() - int64(sw.CreatedUnix)
+
+	// Create TrackedTime
+	tt := &TrackedTime{
+		Created: time.Now(),
+		IssueID: issue.ID,
+		UserID:  user.ID,
+		Time:    timediff,
+	}
+
+	if _, err := e.Insert(tt); err != nil {
+		return err
+	}
+
+	if err := issue.loadRepo(e); err != nil {
+		return err
+	}
+	if _, err := createComment(e, &CreateCommentOptions{
+		Doer:    user,
+		Issue:   issue,
+		Repo:    issue.Repo,
+		Content: SecToTime(timediff),
+		Type:    CommentTypeStopTracking,
+		TimeID:  tt.ID,
+	}); err != nil {
+		return err
+	}
+	_, err = e.Delete(sw)
+	return err
+}
+
+// CreateIssueStopwatch creates a stopwatch if not exist, otherwise return an error
+func CreateIssueStopwatch(user *User, issue *Issue) error {
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := createIssueStopwatch(sess, user, issue); err != nil {
+		return err
+	}
+	return sess.Commit()
+}
+
+func createIssueStopwatch(e *xorm.Session, user *User, issue *Issue) error {
 	if err := issue.loadRepo(e); err != nil {
 		return err
 	}
 
+	// if another stopwatch is running: stop it
+	exists, sw, err := hasUserStopwatch(e, user.ID)
+	if err != nil {
+		return err
+	}
 	if exists {
-		// Create tracked time out of the time difference between start date and actual date
-		timediff := time.Now().Unix() - int64(sw.CreatedUnix)
-
-		// Create TrackedTime
-		tt := &TrackedTime{
-			Created: time.Now(),
-			IssueID: issue.ID,
-			UserID:  user.ID,
-			Time:    timediff,
-		}
-
-		if _, err := e.Insert(tt); err != nil {
-			return err
-		}
-
-		if _, err := createComment(e, &CreateCommentOptions{
-			Doer:    user,
-			Issue:   issue,
-			Repo:    issue.Repo,
-			Content: SecToTime(timediff),
-			Type:    CommentTypeStopTracking,
-			TimeID:  tt.ID,
-		}); err != nil {
-			return err
-		}
-		if _, err := e.Delete(sw); err != nil {
-			return err
-		}
-	} else {
-		// if another stopwatch is running: stop it
-		exists, sw, err := hasUserStopwatch(e, user.ID)
+		issue, err := getIssueByID(e, sw.IssueID)
 		if err != nil {
 			return err
 		}
-		if exists {
-			issue, err := getIssueByID(e, sw.IssueID)
-			if err != nil {
-				return err
-			}
-			if err := createOrStopIssueStopwatch(e, user, issue); err != nil {
-				return err
-			}
-		}
-
-		// Create stopwatch
-		sw = &Stopwatch{
-			UserID:  user.ID,
-			IssueID: issue.ID,
-		}
-
-		if _, err := e.Insert(sw); err != nil {
+		if err := finishIssueStopwatch(e, user, issue); err != nil {
 			return err
 		}
+	}
 
-		if _, err := createComment(e, &CreateCommentOptions{
-			Doer:  user,
-			Issue: issue,
-			Repo:  issue.Repo,
-			Type:  CommentTypeStartTracking,
-		}); err != nil {
-			return err
-		}
+	// Create stopwatch
+	sw = &Stopwatch{
+		UserID:  user.ID,
+		IssueID: issue.ID,
+	}
+
+	if _, err := e.Insert(sw); err != nil {
+		return err
+	}
+
+	if err := issue.loadRepo(e); err != nil {
+		return err
+	}
+
+	if _, err := createComment(e, &CreateCommentOptions{
+		Doer:  user,
+		Issue: issue,
+		Repo:  issue.Repo,
+		Type:  CommentTypeStartTracking,
+	}); err != nil {
+		return err
 	}
 	return nil
 }

models/issue_test.go

@@ -5,7 +5,9 @@
 package models
 
 import (
+	"fmt"
 	"sort"
+	"sync"
 	"testing"
 	"time"
 
@@ -417,3 +419,43 @@ func TestIssue_ResolveMentions(t *testing.T) {
 	// Private repo, whole team
 	testSuccess("user17", "big_test_private_4", "user15", []string{"user17/owners"}, []int64{18})
 }
+
+func TestCorrectIssueStats(t *testing.T) {
+	assert.NoError(t, PrepareTestDatabase())
+
+	// Because the condition is to have chunked database look-ups,
+	// We have to more issues than `maxQueryParameters`, we will insert.
+	// maxQueryParameters + 10 issues into the testDatabase.
+	// Each new issues will have a constant description "Bugs are nasty"
+	// Which will be used later on.
+
+	issueAmount := maxQueryParameters + 10
+
+	var wg sync.WaitGroup
+	for i := 0; i < issueAmount; i++ {
+		wg.Add(1)
+		go func(i int) {
+			testInsertIssue(t, fmt.Sprintf("Issue %d", i+1), "Bugs are nasty", 0)
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+
+	// Now we will get all issueID's that match the "Bugs are nasty" query.
+	total, ids, err := SearchIssueIDsByKeyword("Bugs are nasty", []int64{1}, issueAmount, 0)
+
+	// Just to be sure.
+	assert.NoError(t, err)
+	assert.EqualValues(t, issueAmount, total)
+
+	// Now we will call the GetIssueStats with these IDs and if working,
+	// get the correct stats back.
+	issueStats, err := GetIssueStats(&IssueStatsOptions{
+		RepoID:   1,
+		IssueIDs: ids,
+	})
+
+	// Now check the values.
+	assert.NoError(t, err)
+	assert.EqualValues(t, issueStats.OpenCount, issueAmount)
+}

models/migrations/migrations.go

@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"reflect"
@@ -762,8 +763,14 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		}
 		tableSQL := string(res[0]["sql"])
 
+		// Get the string offset for column definitions: `CREATE TABLE ( column-definitions... )`
+		columnDefinitionsIndex := strings.Index(tableSQL, "(")
+		if columnDefinitionsIndex < 0 {
+			return errors.New("couldn't find column definitions")
+		}
+
 		// Separate out the column definitions
-		tableSQL = tableSQL[strings.Index(tableSQL, "("):]
+		tableSQL = tableSQL[columnDefinitionsIndex:]
 
 		// Remove the required columnNames
 		for _, name := range columnNames {

models/migrations/v161.go

@@ -5,6 +5,8 @@
 package migrations
 
 import (
+	"context"
+
 	"xorm.io/xorm"
 )
 
@@ -40,8 +42,17 @@ func convertTaskTypeToString(x *xorm.Engine) error {
 		return err
 	}
 
+	// to keep the migration could be rerun
+	exist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "hook_task", "type")
+	if err != nil {
+		return err
+	}
+	if !exist {
+		return nil
+	}
+
 	for i, s := range hookTaskTypes {
-		if _, err := x.Exec("UPDATE hook_task set typ = ? where type=?", s, i); err != nil {
+		if _, err := x.Exec("UPDATE hook_task set typ = ? where `type`=?", s, i); err != nil {
 			return err
 		}
 	}

models/migrations/v184.go

@@ -5,6 +5,7 @@
 package migrations
 
 import (
+	"context"
 	"fmt"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -19,6 +20,22 @@ func renameTaskErrorsToMessage(x *xorm.Engine) error {
 		Status int `xorm:"index"`
 	}
 
+	// This migration maybe rerun so that we should check if it has been run
+	messageExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "message")
+	if err != nil {
+		return err
+	}
+
+	if messageExist {
+		errorsExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "errors")
+		if err != nil {
+			return err
+		}
+		if !errorsExist {
+			return nil
+		}
+	}
+
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
@@ -29,6 +46,13 @@ func renameTaskErrorsToMessage(x *xorm.Engine) error {
 		return fmt.Errorf("error on Sync2: %v", err)
 	}
 
+	if messageExist {
+		// if both errors and message exist, drop message at first
+		if err := dropTableColumns(sess, "task", "message"); err != nil {
+			return err
+		}
+	}
+
 	switch {
 	case setting.Database.UseMySQL:
 		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {

models/models.go

@@ -179,16 +179,35 @@ func syncTables() error {
 	return x.StoreEngine("InnoDB").Sync2(tables...)
 }
 
-// NewTestEngine sets a new test xorm.Engine
-func NewTestEngine() (err error) {
+// NewInstallTestEngine creates a new xorm.Engine for testing during install
+//
+// This function will cause the basic database schema to be created
+func NewInstallTestEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) {
 	x, err = GetNewEngine()
 	if err != nil {
-		return fmt.Errorf("Connect to database: %v", err)
+		return fmt.Errorf("failed to connect to database: %w", err)
 	}
 
 	x.SetMapper(names.GonicMapper{})
 	x.SetLogger(NewXORMLogger(!setting.IsProd()))
 	x.ShowSQL(!setting.IsProd())
+
+	x.SetDefaultContext(ctx)
+
+	if err = x.Ping(); err != nil {
+		return err
+	}
+
+	// We have to run migrateFunc here in case the user is re-running installation on a previously created DB.
+	// If we do not then table schemas will be changed and there will be conflicts when the migrations run properly.
+	//
+	// Installation should only be being re-run if users want to recover an old database.
+	// However, we should think carefully about should we support re-install on an installed instance,
+	// as there may be other problems due to secret reinitialization.
+	if err = migrateFunc(x); err != nil {
+		return fmt.Errorf("migrate: %v", err)
+	}
+
 	return syncTables()
 }
 

models/project_issue.go

@@ -115,7 +115,9 @@ func (p *Project) NumClosedIssues() int {
 func (p *Project) NumOpenIssues() int {
 	c, err := x.Table("project_issue").
 		Join("INNER", "issue", "project_issue.issue_id=issue.id").
-		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, false).Count("issue.id")
+		Where("project_issue.project_id=? AND issue.is_closed=?", p.ID, false).
+		Cols("issue_id").
+		Count()
 	if err != nil {
 		return 0
 	}

models/repo.go

@@ -1152,16 +1152,6 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository, overwriteO
 		return fmt.Errorf("recalculateAccesses: %v", err)
 	}
 
-	if u.Visibility == api.VisibleTypePublic && !repo.IsPrivate {
-		// Create/Remove git-daemon-export-ok for git-daemon...
-		daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`)
-		if f, err := os.Create(daemonExportFile); err != nil {
-			log.Error("Failed to create %s: %v", daemonExportFile, err)
-		} else {
-			f.Close()
-		}
-	}
-
 	if setting.Service.AutoWatchNewRepos {
 		if err = watchRepo(ctx.e, doer.ID, repo.ID, true); err != nil {
 			return fmt.Errorf("watchRepo: %v", err)
@@ -1175,6 +1165,46 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository, overwriteO
 	return nil
 }
 
+// CheckDaemonExportOK creates/removes git-daemon-export-ok for git-daemon...
+func (repo *Repository) CheckDaemonExportOK() error {
+	return repo.checkDaemonExportOK(x)
+}
+
+// CheckDaemonExportOKCtx creates/removes git-daemon-export-ok for git-daemon...
+func (repo *Repository) CheckDaemonExportOKCtx(ctx DBContext) error {
+	return repo.checkDaemonExportOK(ctx.e)
+}
+
+func (repo *Repository) checkDaemonExportOK(e Engine) error {
+	if err := repo.getOwner(e); err != nil {
+		return err
+	}
+
+	// Create/Remove git-daemon-export-ok for git-daemon...
+	daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`)
+
+	isExist, err := util.IsExist(daemonExportFile)
+	if err != nil {
+		log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err)
+		return err
+	}
+
+	isPublic := !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePublic
+	if !isPublic && isExist {
+		if err = util.Remove(daemonExportFile); err != nil {
+			log.Error("Failed to remove %s: %v", daemonExportFile, err)
+		}
+	} else if isPublic && !isExist {
+		if f, err := os.Create(daemonExportFile); err != nil {
+			log.Error("Failed to create %s: %v", daemonExportFile, err)
+		} else {
+			f.Close()
+		}
+	}
+
+	return nil
+}
+
 func countRepositories(userID int64, private bool) int64 {
 	sess := x.Where("id > 0")
 
@@ -1324,24 +1354,9 @@ func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err e
 		}
 
 		// Create/Remove git-daemon-export-ok for git-daemon...
-		daemonExportFile := path.Join(repo.RepoPath(), `git-daemon-export-ok`)
-		isExist, err := util.IsExist(daemonExportFile)
-		isPublic := !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePublic
-		if err != nil {
-			log.Error("Unable to check if %s exists. Error: %v", daemonExportFile, err)
+		if err := repo.checkDaemonExportOK(e); err != nil {
 			return err
 		}
-		if !isPublic && isExist {
-			if err = util.Remove(daemonExportFile); err != nil {
-				log.Error("Failed to remove %s: %v", daemonExportFile, err)
-			}
-		} else if isPublic && !isExist {
-			if f, err := os.Create(daemonExportFile); err != nil {
-				log.Error("Failed to create %s: %v", daemonExportFile, err)
-			} else {
-				f.Close()
-			}
-		}
 
 		forkRepos, err := getRepositoriesByForkID(e, repo.ID)
 		if err != nil {

models/u2f.go

@@ -52,7 +52,7 @@ func (list U2FRegistrationList) ToRegistrations() []u2f.Registration {
 	for _, reg := range list {
 		r, err := reg.Parse()
 		if err != nil {
-			log.Fatal("parsing u2f registration: %v", err)
+			log.Error("parsing u2f registration: %v", err)
 			continue
 		}
 		regs = append(regs, *r)

models/u2f_test.go

@@ -5,6 +5,7 @@
 package models
 
 import (
+	"encoding/hex"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -27,6 +28,7 @@ func TestGetU2FRegistrationsByUID(t *testing.T) {
 	assert.NoError(t, PrepareTestDatabase())
 
 	res, err := GetU2FRegistrationsByUID(1)
+
 	assert.NoError(t, err)
 	assert.Len(t, res, 1)
 	assert.Equal(t, "U2F Key", res[0].Name)
@@ -71,3 +73,27 @@ func TestDeleteRegistration(t *testing.T) {
 	assert.NoError(t, DeleteRegistration(reg))
 	AssertNotExistsBean(t, &U2FRegistration{ID: 1})
 }
+
+const validU2FRegistrationResponseHex = "0504b174bc49c7ca254b70d2e5c207cee9cf174820ebd77ea3c65508c26da51b657c1cc6b952f8621697936482da0a6d3d3826a59095daf6cd7c03e2e60385d2f6d9402a552dfdb7477ed65fd84133f86196010b2215b57da75d315b7b9e8fe2e3925a6019551bab61d16591659cbaf00b4950f7abfe6660e2e006f76868b772d70c253082013c3081e4a003020102020a47901280001155957352300a06082a8648ce3d0403023017311530130603550403130c476e756262792050696c6f74301e170d3132303831343138323933325a170d3133303831343138323933325a3031312f302d0603550403132650696c6f74476e756262792d302e342e312d34373930313238303030313135353935373335323059301306072a8648ce3d020106082a8648ce3d030107034200048d617e65c9508e64bcc5673ac82a6799da3c1446682c258c463fffdf58dfd2fa3e6c378b53d795c4a4dffb4199edd7862f23abaf0203b4b8911ba0569994e101300a06082a8648ce3d0403020347003044022060cdb6061e9c22262d1aac1d96d8c70829b2366531dda268832cb836bcd30dfa0220631b1459f09e6330055722c8d89b7f48883b9089b88d60d1d9795902b30410df304502201471899bcc3987e62e8202c9b39c33c19033f7340352dba80fcab017db9230e402210082677d673d891933ade6f617e5dbde2e247e70423fd5ad7804a6d3d3961ef871"
+
+func TestToRegistrations_SkipInvalidItemsWithoutCrashing(t *testing.T) {
+	regKeyRaw, _ := hex.DecodeString(validU2FRegistrationResponseHex)
+	regs := U2FRegistrationList{
+		&U2FRegistration{ID: 1},
+		&U2FRegistration{ID: 2, Name: "U2F Key", UserID: 2, Counter: 0, Raw: regKeyRaw, CreatedUnix: 946684800, UpdatedUnix: 946684800},
+	}
+
+	actual := regs.ToRegistrations()
+	assert.Len(t, actual, 1)
+}
+
+func TestToRegistrations(t *testing.T) {
+	regKeyRaw, _ := hex.DecodeString(validU2FRegistrationResponseHex)
+	regs := U2FRegistrationList{
+		&U2FRegistration{ID: 1, Name: "U2F Key", UserID: 1, Counter: 0, Raw: regKeyRaw, CreatedUnix: 946684800, UpdatedUnix: 946684800},
+		&U2FRegistration{ID: 2, Name: "U2F Key", UserID: 2, Counter: 0, Raw: regKeyRaw, CreatedUnix: 946684800, UpdatedUnix: 946684800},
+	}
+
+	actual := regs.ToRegistrations()
+	assert.Len(t, actual, 2)
+}

models/user.go

@@ -77,9 +77,6 @@ var (
 	// ErrEmailNotActivated e-mail address has not been activated error
 	ErrEmailNotActivated = errors.New("E-mail address has not been activated")
 
-	// ErrUserNameIllegal user name contains illegal characters error
-	ErrUserNameIllegal = errors.New("User name contains illegal characters")
-
 	// ErrLoginSourceNotActived login source is not actived error
 	ErrLoginSourceNotActived = errors.New("Login source is not actived")
 
@@ -1072,18 +1069,46 @@ func validateUser(u *User) error {
 	return ValidateEmail(u.Email)
 }
 
-func updateUser(e Engine, u *User) error {
+func updateUser(e Engine, u *User, changePrimaryEmail bool) error {
 	if err := validateUser(u); err != nil {
 		return err
 	}
 
+	if changePrimaryEmail {
+		var emailAddress EmailAddress
+		has, err := e.Where("lower_email=?", strings.ToLower(u.Email)).Get(&emailAddress)
+		if err != nil {
+			return err
+		}
+		if !has {
+			// 1. Update old primary email
+			if _, err = e.Where("uid=? AND is_primary=?", u.ID, true).Cols("is_primary").Update(&EmailAddress{
+				IsPrimary: false,
+			}); err != nil {
+				return err
+			}
+
+			emailAddress.Email = u.Email
+			emailAddress.UID = u.ID
+			emailAddress.IsActivated = true
+			emailAddress.IsPrimary = true
+			if _, err := e.Insert(&emailAddress); err != nil {
+				return err
+			}
+		} else if _, err := e.ID(emailAddress).Cols("is_primary").Update(&EmailAddress{
+			IsPrimary: true,
+		}); err != nil {
+			return err
+		}
+	}
+
 	_, err := e.ID(u.ID).AllCols().Update(u)
 	return err
 }
 
 // UpdateUser updates user's information.
-func UpdateUser(u *User) error {
-	return updateUser(x, u)
+func UpdateUser(u *User, changePrimaryEmail bool) error {
+	return updateUser(x, u, changePrimaryEmail)
 }
 
 // UpdateUserCols update user according special columns
@@ -1112,7 +1137,7 @@ func UpdateUserSetting(u *User) (err error) {
 			return err
 		}
 	}
-	if err = updateUser(sess, u); err != nil {
+	if err = updateUser(sess, u, false); err != nil {
 		return err
 	}
 	return sess.Commit()

models/user_heatmap_test.go

@@ -7,6 +7,9 @@ package models
 import (
 	"fmt"
 	"testing"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/stretchr/testify/assert"
@@ -37,6 +40,10 @@ func TestGetUserHeatmapDataByUser(t *testing.T) {
 	// Prepare
 	assert.NoError(t, PrepareTestDatabase())
 
+	// Mock time
+	timeutil.Set(time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC))
+	defer timeutil.Unset()
+
 	for i, tc := range testCases {
 		user := AssertExistsAndLoadBean(t, &User{ID: tc.userID}).(*User)
 

models/user_test.go

@@ -475,17 +475,17 @@ func TestUpdateUser(t *testing.T) {
 	user := AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)
 
 	user.KeepActivityPrivate = true
-	assert.NoError(t, UpdateUser(user))
+	assert.NoError(t, UpdateUser(user, false))
 	user = AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)
 	assert.True(t, user.KeepActivityPrivate)
 
 	setting.Service.AllowedUserVisibilityModesSlice = []bool{true, false, false}
 	user.KeepActivityPrivate = false
 	user.Visibility = structs.VisibleTypePrivate
-	assert.Error(t, UpdateUser(user))
+	assert.Error(t, UpdateUser(user, false))
 	user = AssertExistsAndLoadBean(t, &User{ID: 2}).(*User)
 	assert.True(t, user.KeepActivityPrivate)
 
 	user.Email = "no mail@mail.org"
-	assert.Error(t, UpdateUser(user))
+	assert.Error(t, UpdateUser(user, true))
 }

modules/auth/pam/pam.go

@@ -1,3 +1,4 @@
+//go:build pam
 // +build pam
 
 // Copyright 2014 The Gogs Authors. All rights reserved.

modules/auth/pam/pam_stub.go

@@ -1,9 +1,10 @@
-// +build !pam
-
 // Copyright 2014 The Gogs Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !pam
+// +build !pam
+
 package pam
 
 import (

modules/auth/pam/pam_test.go

@@ -1,3 +1,4 @@
+//go:build pam
 // +build pam
 
 // Copyright 2021 The Gitea Authors. All rights reserved.

modules/charset/charset.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gogs/chardet"
 	"golang.org/x/net/html/charset"
@@ -26,9 +27,9 @@ var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
 func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
 	var buf = make([]byte, 2048)
-	n, err := rd.Read(buf)
+	n, err := util.ReadAtMost(rd, buf)
 	if err != nil {
-		return rd
+		return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
 	}
 
 	charsetLabel, err := DetectEncoding(buf[:n])

modules/context/repo.go

@@ -58,6 +58,7 @@ type Repository struct {
 	Commit       *git.Commit
 	Tag          *git.Tag
 	GitRepo      *git.Repository
+	RefName      string
 	BranchName   string
 	TagName      string
 	TreePath     string
@@ -190,9 +191,9 @@ func (r *Repository) BranchNameSubURL() string {
 	case r.IsViewBranch:
 		return "branch/" + r.BranchName
 	case r.IsViewTag:
-		return "tag/" + r.BranchName
+		return "tag/" + r.TagName
 	case r.IsViewCommit:
-		return "commit/" + r.BranchName
+		return "commit/" + r.CommitID
 	}
 	log.Error("Unknown view type for repo: %v", r)
 	return ""
@@ -345,7 +346,7 @@ func repoAssignment(ctx *Context, repo *models.Repository) {
 	}
 
 	// Check access.
-	if ctx.Repo.Permission.AccessMode == models.AccessModeNone {
+	if !ctx.Repo.Permission.HasAccess() {
 		if ctx.Query("go-get") == "1" {
 			EarlyResponseForGoGetMeta(ctx)
 			return
@@ -562,8 +563,6 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 	ctx.Data["Branches"] = brs
 	ctx.Data["BranchesCount"] = len(brs)
 
-	ctx.Data["TagName"] = ctx.Repo.TagName
-
 	// If not branch selected, try default one.
 	// If default branch doesn't exists, fall back to some other branch.
 	if len(ctx.Repo.BranchName) == 0 {
@@ -572,9 +571,9 @@ func RepoAssignment(ctx *Context) (cancel context.CancelFunc) {
 		} else if len(brs) > 0 {
 			ctx.Repo.BranchName = brs[0]
 		}
+		ctx.Repo.RefName = ctx.Repo.BranchName
 	}
 	ctx.Data["BranchName"] = ctx.Repo.BranchName
-	ctx.Data["CommitID"] = ctx.Repo.CommitID
 
 	// People who have push access or have forked repository can propose a new pull request.
 	canPush := ctx.Repo.CanWrite(models.UnitTypeCode) || (ctx.IsSigned && ctx.User.HasForkedRepo(ctx.Repo.Repository.ID))
@@ -759,7 +758,6 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 		// Get default branch.
 		if len(ctx.Params("*")) == 0 {
 			refName = ctx.Repo.Repository.DefaultBranch
-			ctx.Repo.BranchName = refName
 			if !ctx.Repo.GitRepo.IsBranchExist(refName) {
 				brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0)
 				if err != nil {
@@ -773,6 +771,8 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 				}
 				refName = brs[0]
 			}
+			ctx.Repo.RefName = refName
+			ctx.Repo.BranchName = refName
 			ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName)
 			if err != nil {
 				ctx.ServerError("GetBranchCommit", err)
@@ -783,9 +783,10 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 
 		} else {
 			refName = getRefName(ctx, refType)
-			ctx.Repo.BranchName = refName
+			ctx.Repo.RefName = refName
 			if refType.RefTypeIncludesBranches() && ctx.Repo.GitRepo.IsBranchExist(refName) {
 				ctx.Repo.IsViewBranch = true
+				ctx.Repo.BranchName = refName
 
 				ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName)
 				if err != nil {
@@ -796,6 +797,8 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 
 			} else if refType.RefTypeIncludesTags() && ctx.Repo.GitRepo.IsTagExist(refName) {
 				ctx.Repo.IsViewTag = true
+				ctx.Repo.TagName = refName
+
 				ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName)
 				if err != nil {
 					ctx.ServerError("GetTagCommit", err)
@@ -837,6 +840,7 @@ func RepoRefByType(refType RepoRefType, ignoreNotExistErr ...bool) func(*Context
 
 		ctx.Data["BranchName"] = ctx.Repo.BranchName
 		ctx.Data["BranchNameSubURL"] = ctx.Repo.BranchNameSubURL()
+		ctx.Data["TagName"] = ctx.Repo.TagName
 		ctx.Data["CommitID"] = ctx.Repo.CommitID
 		ctx.Data["TreePath"] = ctx.Repo.TreePath
 		ctx.Data["IsViewBranch"] = ctx.Repo.IsViewBranch

modules/convert/git_commit.go

@@ -147,8 +147,9 @@ func ToCommit(repo *models.Repository, commit *git.Commit, userCache map[string]
 
 	return &api.Commit{
 		CommitMeta: &api.CommitMeta{
-			URL: repo.APIURL() + "/git/commits/" + commit.ID.String(),
-			SHA: commit.ID.String(),
+			URL:     repo.APIURL() + "/git/commits/" + commit.ID.String(),
+			SHA:     commit.ID.String(),
+			Created: commit.Committer.When,
 		},
 		HTMLURL: repo.HTMLURL() + "/commit/" + commit.ID.String(),
 		RepoCommit: &api.RepoCommit{
@@ -169,8 +170,9 @@ func ToCommit(repo *models.Repository, commit *git.Commit, userCache map[string]
 			},
 			Message: commit.Message(),
 			Tree: &api.CommitMeta{
-				URL: repo.APIURL() + "/git/trees/" + commit.ID.String(),
-				SHA: commit.ID.String(),
+				URL:     repo.APIURL() + "/git/trees/" + commit.ID.String(),
+				SHA:     commit.ID.String(),
+				Created: commit.Committer.When,
 			},
 		},
 		Author:    apiAuthor,

modules/csv/csv.go

@@ -28,32 +28,24 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
 }
 
 // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
+// Reads at most 10k bytes.
 func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
 	var data = make([]byte, 1e4)
-	size, err := rd.Read(data)
+	size, err := util.ReadAtMost(rd, data)
 	if err != nil {
 		return nil, err
 	}
 
-	delimiter := guessDelimiter(data[:size])
-
-	var newInput io.Reader
-	if size < 1e4 {
-		newInput = bytes.NewReader(data[:size])
-	} else {
-		newInput = io.MultiReader(bytes.NewReader(data), rd)
-	}
-
-	return CreateReader(newInput, delimiter), nil
+	return CreateReader(
+		io.MultiReader(bytes.NewReader(data[:size]), rd),
+		guessDelimiter(data[:size]),
+	), nil
 }
 
 // guessDelimiter scores the input CSV data against delimiters, and returns the best match.
-// Reads at most 10k bytes & 10 lines.
 func guessDelimiter(data []byte) rune {
 	maxLines := 10
-	maxBytes := util.Min(len(data), 1e4)
-	text := string(data[:maxBytes])
-	text = quoteRegexp.ReplaceAllLiteralString(text, "")
+	text := quoteRegexp.ReplaceAllLiteralString(string(data), "")
 	lines := strings.SplitN(text, "\n", maxLines+1)
 	lines = lines[:util.Min(maxLines, len(lines))]
 

modules/git/blob.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 
 	"code.gitea.io/gitea/modules/typesniffer"
+	"code.gitea.io/gitea/modules/util"
 )
 
 // This file contains common functions between the gogit and !gogit variants for git Blobs
@@ -29,7 +30,7 @@ func (b *Blob) GetBlobContent() (string, error) {
 	}
 	defer dataRc.Close()
 	buf := make([]byte, 1024)
-	n, _ := dataRc.Read(buf)
+	n, _ := util.ReadAtMost(dataRc, buf)
 	buf = buf[:n]
 	return string(buf), nil
 }

modules/git/blob_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/blob_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/command_test.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build race
 // +build race
 
 package git

modules/git/commit_convert_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/commit_info_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/commit_info_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/git.go

@@ -188,6 +188,12 @@ func Init(ctx context.Context) error {
 			return err
 		}
 	}
+	if setting.Git.DisableCoreProtectNTFS {
+		if err := checkAndSetConfig("core.protectntfs", "false", true); err != nil {
+			return err
+		}
+		GlobalCommandArgs = append(GlobalCommandArgs, "-c", "core.protectntfs=false")
+	}
 	return nil
 }
 

modules/git/last_commit_cache_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/last_commit_cache_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/notes_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/notes_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/parse_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/parse_gogit_test.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/parse_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/parse_nogogit_test.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/pipeline/lfs.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package pipeline

modules/git/pipeline/lfs_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package pipeline

modules/git/repo_base_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_base_nogogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_blob_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_blob_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_branch_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_branch_nogogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_commit_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_commit_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_commitgraph_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_language_stats_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_language_stats_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_ref_gogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_ref_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_tag_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_tag_nogogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/repo_tree_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/repo_tree_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/sha1_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/sha1_nogogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/signature_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/signature_nogogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/submodule.go

@@ -52,9 +52,7 @@ func getRefURL(refURL, urlPrefix, repoFullName, sshDomain string) string {
 		urlPrefixHostname = prefixURL.Host
 	}
 
-	if strings.HasSuffix(urlPrefix, "/") {
-		urlPrefix = urlPrefix[:len(urlPrefix)-1]
-	}
+	urlPrefix = strings.TrimSuffix(urlPrefix, "/")
 
 	// FIXME: Need to consider branch - which will require changes in modules/git/commit.go:GetSubModules
 	// Relative url prefix check (according to git submodule documentation)

modules/git/tree_blob_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/tree_blob_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/tree_entry_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/tree_entry_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/git/tree_entry_test.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/tree_gogit.go

@@ -3,6 +3,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build gogit
 // +build gogit
 
 package git

modules/git/tree_nogogit.go

@@ -2,6 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !gogit
 // +build !gogit
 
 package git

modules/gitgraph/graph_models.go

@@ -217,11 +217,9 @@ func newRefsFromRefNames(refNames []byte) []git.Reference {
 			continue
 		}
 		refName := string(refNameBytes)
-		if strings.HasPrefix(refName, "tag: ") {
-			refName = strings.TrimPrefix(refName, "tag: ")
-		} else if strings.HasPrefix(refName, "HEAD -> ") {
-			refName = strings.TrimPrefix(refName, "HEAD -> ")
-		}
+		refName = strings.TrimPrefix(refName, "tag: ")
+		refName = strings.TrimPrefix(refName, "HEAD -> ")
+
 		refs = append(refs, git.Reference{
 			Name: refName,
 		})

modules/graceful/manager_unix.go

@@ -1,9 +1,10 @@
-// +build !windows
-
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+//go:build !windows
+// +build !windows
+
 package graceful
 
 import (

modules/graceful/manager_windows.go

@@ -1,10 +1,11 @@
-// +build windows
-
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 // This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
 
+//go:build windows
+// +build windows
+
 package graceful
 
 import (

modules/graceful/net_unix.go

@@ -1,10 +1,11 @@
-// +build !windows
-
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 // This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
 
+//go:build !windows
+// +build !windows
+
 package graceful
 
 import (

modules/graceful/net_windows.go

@@ -1,10 +1,11 @@
-// +build windows
-
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 // This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
 
+//go:build windows
+// +build windows
+
 package graceful
 
 import "net"

modules/graceful/restart_unix.go

@@ -1,10 +1,11 @@
-// +build !windows
-
 // Copyright 2019 The Gitea Authors. All rights reserved.
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 // This code is heavily inspired by the archived gofacebook/gracenet/net.go handler
 
+//go:build !windows
+// +build !windows
+
 package graceful
 
 import (

modules/graceful/server.go

@@ -229,7 +229,7 @@ func (wl *wrappedListener) Accept() (net.Conn, error) {
 
 	closed := int32(0)
 
-	c = wrappedConn{
+	c = &wrappedConn{
 		Conn:                 c,
 		server:               wl.server,
 		closed:               &closed,
@@ -264,7 +264,7 @@ type wrappedConn struct {
 	perWritePerKbTimeout time.Duration
 }
 
-func (w wrappedConn) Write(p []byte) (n int, err error) {
+func (w *wrappedConn) Write(p []byte) (n int, err error) {
 	if w.perWriteTimeout > 0 {
 		minTimeout := time.Duration(len(p)/1024) * w.perWritePerKbTimeout
 		minDeadline := time.Now().Add(minTimeout).Add(w.perWriteTimeout)
@@ -278,7 +278,7 @@ func (w wrappedConn) Write(p []byte) (n int, err error) {
 	return w.Conn.Write(p)
 }
 
-func (w wrappedConn) Close() error {
+func (w *wrappedConn) Close() error {
 	if atomic.CompareAndSwapInt32(w.closed, 0, 1) {
 		defer func() {
 			if err := recover(); err != nil {

modules/hostmatcher/hostmatcher.go

@@ -0,0 +1,94 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package hostmatcher
+
+import (
+	"net"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// HostMatchList is used to check if a host or IP is in a list.
+// If you only need to do wildcard matching, consider to use modules/matchlist
+type HostMatchList struct {
+	hosts  []string
+	ipNets []*net.IPNet
+}
+
+// MatchBuiltinAll all hosts are matched
+const MatchBuiltinAll = "*"
+
+// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched
+const MatchBuiltinExternal = "external"
+
+// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+const MatchBuiltinPrivate = "private"
+
+// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+const MatchBuiltinLoopback = "loopback"
+
+// ParseHostMatchList parses the host list HostMatchList
+func ParseHostMatchList(hostList string) *HostMatchList {
+	hl := &HostMatchList{}
+	for _, s := range strings.Split(hostList, ",") {
+		s = strings.ToLower(strings.TrimSpace(s))
+		if s == "" {
+			continue
+		}
+		_, ipNet, err := net.ParseCIDR(s)
+		if err == nil {
+			hl.ipNets = append(hl.ipNets, ipNet)
+		} else {
+			hl.hosts = append(hl.hosts, s)
+		}
+	}
+	return hl
+}
+
+// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list
+func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool {
+	var matched bool
+	host = strings.ToLower(host)
+	ipStr := ip.String()
+loop:
+	for _, hostInList := range hl.hosts {
+		switch hostInList {
+		case "":
+			continue
+		case MatchBuiltinAll:
+			matched = true
+			break loop
+		case MatchBuiltinExternal:
+			if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched {
+				break loop
+			}
+		case MatchBuiltinPrivate:
+			if matched = util.IsIPPrivate(ip); matched {
+				break loop
+			}
+		case MatchBuiltinLoopback:
+			if matched = ip.IsLoopback(); matched {
+				break loop
+			}
+		default:
+			if matched, _ = filepath.Match(hostInList, host); matched {
+				break loop
+			}
+			if matched, _ = filepath.Match(hostInList, ipStr); matched {
+				break loop
+			}
+		}
+	}
+	if !matched {
+		for _, ipNet := range hl.ipNets {
+			if matched = ipNet.Contains(ip); matched {
+				break
+			}
+		}
+	}
+	return matched
+}

modules/hostmatcher/hostmatcher_test.go

@@ -0,0 +1,119 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package hostmatcher
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHostOrIPMatchesList(t *testing.T) {
+	type tc struct {
+		host     string
+		ip       net.IP
+		expected bool
+	}
+
+	// for IPv6: "::1" is loopback, "fd00::/8" is private
+
+	hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24")
+	cases := []tc{
+		{"", net.IPv4zero, false},
+		{"", net.IPv6zero, false},
+
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("::1"), false},
+
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("fd00::1"), true},
+
+		{"", net.ParseIP("8.8.8.8"), true},
+		{"", net.ParseIP("1001::1"), true},
+
+		{"mydomain.com", net.IPv4zero, false},
+		{"sub.mydomain.com", net.IPv4zero, true},
+
+		{"", net.ParseIP("169.254.1.1"), true},
+		{"", net.ParseIP("169.254.2.2"), false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("loopback")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), true},
+		{"", net.ParseIP("10.0.1.1"), false},
+		{"", net.ParseIP("192.168.1.1"), false},
+		{"", net.ParseIP("8.8.8.8"), false},
+
+		{"", net.ParseIP("::1"), true},
+		{"", net.ParseIP("fd00::1"), false},
+		{"", net.ParseIP("1000::1"), false},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("private")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("8.8.8.8"), false},
+
+		{"", net.ParseIP("::1"), false},
+		{"", net.ParseIP("fd00::1"), true},
+		{"", net.ParseIP("1000::1"), false},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("external")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("10.0.1.1"), false},
+		{"", net.ParseIP("192.168.1.1"), false},
+		{"", net.ParseIP("8.8.8.8"), true},
+
+		{"", net.ParseIP("::1"), false},
+		{"", net.ParseIP("fd00::1"), false},
+		{"", net.ParseIP("1000::1"), true},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("*")
+	cases = []tc{
+		{"", net.IPv4zero, true},
+		{"", net.ParseIP("127.0.0.1"), true},
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("8.8.8.8"), true},
+
+		{"", net.ParseIP("::1"), true},
+		{"", net.ParseIP("fd00::1"), true},
+		{"", net.ParseIP("1000::1"), true},
+
+		{"mydomain.com", net.IPv4zero, true},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+}