fix: get valid UTF-8 for diff_hunk and composer readme (#39932)

* Fixes #39929
* Fixes #39994

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/context/base.go

@@ -114,7 +114,7 @@ func (b *Base) HTTPError(status int, contents ...string) {
 func (b *Base) JSON(status int, content any) {
 	b.Resp.Header().Set("Content-Type", "application/json;charset=utf-8")
 	b.Resp.WriteHeader(status)
-	if err := json.MarshalWrite(b.Resp, content); err != nil {
+	if err := json.MarshalWriteRelaxed(b.Resp, content); err != nil {
 		log.Error("Render JSON failed: %v", err)
 	}
 }

services/convert/git_commit.go

@@ -189,17 +189,17 @@ func ToCommit(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep
 
 	// Retrieve files affected by the commit
 	if opts.Files {
-		fileStatus, err := git.GetCommitFileStatus(ctx, repo, commit.ID.String())
+		fileStatusSummary, err := git.GetCommitFileStatus(ctx, repo, commit.ID.String())
 		if err != nil {
 			return nil, err
 		}
 
-		affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatus.Added)+len(fileStatus.Removed)+len(fileStatus.Modified))
-		for filestatus, files := range map[string][]string{"added": fileStatus.Added, "removed": fileStatus.Removed, "modified": fileStatus.Modified} {
+		affectedFileList := make([]*api.CommitAffectedFiles, 0, len(fileStatusSummary.Added)+len(fileStatusSummary.Removed)+len(fileStatusSummary.Modified))
+		for fileStatus, files := range map[string][]string{"added": fileStatusSummary.Added, "removed": fileStatusSummary.Removed, "modified": fileStatusSummary.Modified} {
 			for _, filename := range files {
 				affectedFileList = append(affectedFileList, &api.CommitAffectedFiles{
 					Filename: filename,
-					Status:   filestatus,
+					Status:   fileStatus,
 				})
 			}
 		}

services/convert/pull_review.go

@@ -6,10 +6,13 @@ package convert
 import (
 	"context"
 	"strings"
+	"unicode/utf8"
 
 	issues_model "gitea.dev/models/issues"
 	user_model "gitea.dev/models/user"
+	"gitea.dev/modules/charset"
 	api "gitea.dev/modules/structs"
+	"gitea.dev/modules/util"
 )
 
 // ToPullReview convert a review to api format
@@ -127,9 +130,13 @@ func ToPullReviewComment(ctx context.Context, comment *issues_model.Comment, doe
 }
 
 func patch2diff(patch string) string {
-	split := strings.Split(patch, "\n@@")
-	if len(split) == 2 {
-		return "@@" + split[1]
+	idx := strings.Index(patch, "\n@@")
+	if idx == -1 {
+		return ""
 	}
-	return ""
+	diffContent := patch[idx+1:]
+	// Although there might be an edge case that deleted lines and added lines are in different encodings,
+	// it's not worth to make things more complicated at the moment, and most code in modern codebases should be written in UTF-8.
+	hunk := charset.ToUTF8(util.UnsafeStringToBytes(diffContent), charset.ConvertOpts{ErrorReplacement: []byte(string(utf8.RuneError))})
+	return util.UnsafeBytesToString(hunk)
 }

services/convert/pull_review_test.go

@@ -49,3 +49,7 @@ func Test_ToPullReview(t *testing.T) {
 		assert.Len(t, prList, 1)
 	})
 }
+
+func TestPatch2DiffInvalidUTF8(t *testing.T) {
+	assert.Equal(t, "@@ -1 +1 @@\n+caf�", patch2diff("diff --git a/a b/a\n@@ -1 +1 @@\n+caf\xe9"))
+}

services/websocket/events.go

@@ -38,7 +38,7 @@ func MakeUserEventMessage(eventType string, eventData any) []byte {
 	buf := &bytes.Buffer{}
 	buf.WriteString(eventType)
 	buf.WriteByte('\n')
-	err := json.MarshalWrite(buf, &UserEventMessage[any]{EventType: eventType, EventData: eventData})
+	err := json.MarshalWriteRelaxed(buf, &UserEventMessage[any]{EventType: eventType, EventData: eventData})
 	payloadBytes := bytes.TrimSuffix(buf.Bytes(), []byte("\n")) // json v1 adds extra "\n" but we don't want it
 	if err != nil {
 		setting.PanicInDevOrTesting("websocket: marshal event: %v", err)