fix: get valid UTF-8 for diff_hunk and composer readme (#39932)

* Fixes #39929
* Fixes #39994

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/charset/charset.go

@@ -52,7 +52,7 @@ func Lookup(label string) (e encoding.Encoding, name string) {
 
 type ConvertOpts struct {
 	KeepBOM           bool
-	ErrorReplacement  []byte
+	ErrorReplacement  []byte // leave it empty to remove the invalid bytes
 	ErrorReturnOrigin bool
 }
 
@@ -99,7 +99,12 @@ func ToUTF8DropErrors(content []byte) []byte {
 func ToUTF8(content []byte, opts ConvertOpts) []byte {
 	charsetLabel, _ := DetectEncoding(content)
 	if charsetLabel == "UTF-8" {
-		return maybeRemoveBOM(content, opts)
+		// even if the detected charset is UTF-8, the content may contain invalid UTF-8 sequences, so need to convert it to valid UTF-8
+		content = maybeRemoveBOM(content, opts)
+		if opts.ErrorReturnOrigin {
+			return content
+		}
+		return bytes.ToValidUTF8(content, opts.ErrorReplacement)
 	}
 
 	encoding, _ := Lookup(charsetLabel)
@@ -123,11 +128,7 @@ func ToUTF8(content []byte, opts ConvertOpts) []byte {
 		if opts.ErrorReturnOrigin {
 			return content
 		}
-		if opts.ErrorReplacement == nil {
-			decoded = append(decoded, content[idx+n])
-		} else {
-			decoded = append(decoded, opts.ErrorReplacement...)
-		}
+		decoded = append(decoded, opts.ErrorReplacement...)
 		idx += n + 1
 	}
 	return maybeRemoveBOM(decoded, opts)

modules/charset/charset_test.go

@@ -171,6 +171,8 @@ func TestToUTF8DropErrors(t *testing.T) {
 
 	res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00})
 	assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res)
+
+	assert.Equal(t, "caf ", string(ToUTF8DropErrors([]byte("caf\xe9"))))
 }
 
 func TestDetectEncoding(t *testing.T) {