fix: get valid UTF-8 for diff_hunk and composer readme (#39932)

* Fixes #39929
* Fixes #39994

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/json/json.go

@@ -23,7 +23,7 @@ type Decoder interface {
 // Interface represents an interface to handle json data
 type Interface interface {
 	Marshal(v any) ([]byte, error)
-	MarshalWrite(w io.Writer, v any) error
+	MarshalWriteRelaxed(w io.Writer, v any) error
 	Unmarshal(data []byte, v any) error
 	NewEncoder(writer io.Writer) Encoder
 	NewDecoder(reader io.Reader) Decoder
@@ -37,9 +37,10 @@ func Marshal(v any) ([]byte, error) {
 	return DefaultJSONHandler.Marshal(v)
 }
 
-// MarshalWrite writes the JSON encoding of v to the given writer
-func MarshalWrite(w io.Writer, v any) error {
-	return DefaultJSONHandler.MarshalWrite(w, v)
+// MarshalWriteRelaxed writes the JSON encoding of v to the given writer
+// It tolerates invalid UTF-8 in strings and replaces them with the Unicode replacement character U+FFFD.
+func MarshalWriteRelaxed(w io.Writer, v any) error {
+	return DefaultJSONHandler.MarshalWriteRelaxed(w, v)
 }
 
 // Unmarshal decodes object from bytes

modules/json/jsonv2.go

@@ -13,6 +13,7 @@ import (
 
 type JSONv2 struct {
 	marshalOptions                  jsonv2.Options
+	marshalRelaxedOptions           jsonv2.Options
 	marshalKeepOptionalEmptyOptions jsonv2.Options
 	unmarshalOptions                jsonv2.Options
 	unmarshalCaseInsensitiveOptions jsonv2.Options
@@ -26,6 +27,8 @@ func init() {
 		jsonv2.FormatNilMapAsNull(true),
 	}
 	jsonV2.marshalOptions = jsonv2.JoinOptions(commonMarshalOptions...)
+	// when handling user's data, we want to allow invalid UTF-8 to be replaced with the replacement character, rather than returning an error.
+	jsonV2.marshalRelaxedOptions = jsonv2.JoinOptions(append(commonMarshalOptions, jsontext.AllowInvalidUTF8(true))...)
 	jsonV2.unmarshalOptions = jsonv2.DefaultOptionsV2()
 
 	// By default, "json/v2" omitempty removes all `""` empty strings, no matter where it comes from.
@@ -53,8 +56,8 @@ func (j *JSONv2) Marshal(v any) ([]byte, error) {
 	return jsonv2.Marshal(v, j.marshalOptions)
 }
 
-func (j *JSONv2) MarshalWrite(w io.Writer, v any) error {
-	return jsonv2.MarshalWrite(w, v, j.marshalOptions)
+func (j *JSONv2) MarshalWriteRelaxed(w io.Writer, v any) error {
+	return jsonv2.MarshalWrite(w, v, j.marshalRelaxedOptions)
 }
 
 func (j *JSONv2) Unmarshal(data []byte, v any) error {