Files
gitea/modules/highlight/highlight.go
T
Wei-Lun Liandsilverwind 901859a2e2 perf(highlight): reuse one chroma HTML formatter (#40418)
`RenderCodeByLexer` created a new chroma HTML formatter on every call,
so the formatter rebuilt the CSS map of the style each time. That runs
for every fenced code block in markdown (since
https://github.com/go-gitea/gitea/pull/39751) and for every line when
the diff view highlights line by line (files over 256 KiB, or after the
full-file highlight time limit). The formatter caches that map behind a
mutex and is otherwise read-only, so one shared instance is enough.
Output is unchanged.

Local Go benchmarks, i7-9700 on Windows 11, base and patched run
alternately 6 times, medians:

| | before | after |
|---|---|---|
| 1000 lines of `services/gitdiff/gitdiff.go`, one call per line | 115.3
ms, 802k allocs | 40.8 ms, 180k allocs |
| 3-line shell code block | 142 µs, 889 allocs | 66 µs, 267 allocs |
| Markdown render of README + CONTRIBUTING + 3 docs (45 KB, 31 code
blocks) | 16.0 ms, 63.8k allocs | 14.4 ms, 45.8k allocs |

This is separate from https://github.com/go-gitea/gitea/pull/40394,
which caches highlighted lines but still creates the formatter on each
call.

Assisted-by: Claude Code:claude-opus-5-5

---------

Co-authored-by: silverwind <me@silverwind.io>
2026-11-18 08:20:24 +00:00

228 lines
7.6 KiB
Go

// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package highlight
import (
"bytes"
gohtml "html"
"html/template"
"strings"
"sync"
"gitea.dev/modules/htmlutil"
"gitea.dev/modules/log"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
"github.com/alecthomas/chroma/v2"
chromahtml "github.com/alecthomas/chroma/v2/formatters/html"
"github.com/alecthomas/chroma/v2/lexers"
"github.com/alecthomas/chroma/v2/styles"
)
// don't highlight files larger than this many bytes for performance purposes
const sizeLimit = 1024 * 1024
type globalVarsType struct {
highlightMapping map[string]string
githubStyles *chroma.Style
htmlFormatter *chromahtml.Formatter // shared because it caches the CSS built from the style
}
var (
globalVarsMu sync.Mutex
globalVarsPtr *globalVarsType
)
func globalVars() *globalVarsType {
// in the future, the globalVars might need to be re-initialized when settings change, so don't use sync.Once here
globalVarsMu.Lock()
defer globalVarsMu.Unlock()
if globalVarsPtr == nil {
globalVarsPtr = &globalVarsType{}
globalVarsPtr.githubStyles = styles.Get("github")
globalVarsPtr.highlightMapping = setting.GetHighlightMapping()
globalVarsPtr.htmlFormatter = chromahtml.New(chromahtml.WithClasses(true),
chromahtml.WithLineNumbers(false),
chromahtml.PreventSurroundingPre(true),
)
}
return globalVarsPtr
}
// UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags
// It always includes '\n', '\n' can appear at the end of each line or in the middle of HTML tags
// The '\n' is necessary for copying code from web UI to preserve original code lines
// ATTENTION: It uses the unsafe conversion between string and []byte for performance reason
// DO NOT make any modification to the returned [][]byte slice items
func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) {
buf := util.UnsafeStringToBytes(string(code))
lineCount := bytes.Count(buf, []byte("\n")) + 1
ret = make([][]byte, 0, lineCount)
nlTagClose := []byte("\n</")
for {
pos := bytes.IndexByte(buf, '\n')
if pos == -1 {
if len(buf) > 0 {
ret = append(ret, buf)
}
return ret
}
// Chroma highlighting output sometimes have "</span>" right after \n, sometimes before.
// * "<span>text\n</span>"
// * "<span>text</span>\n"
if bytes.HasPrefix(buf[pos:], nlTagClose) {
pos1 := bytes.IndexByte(buf[pos:], '>')
if pos1 != -1 {
pos += pos1
}
}
ret = append(ret, buf[:pos+1])
buf = buf[pos+1:]
}
}
func htmlEscape(code string) template.HTML {
return template.HTML(gohtml.EscapeString(code))
}
func normalizeLexer(lexer chroma.Lexer) chroma.Lexer {
if lexer != lexers.Fallback {
lexer = chroma.Coalesce(lexer)
}
return lexer
}
// RenderCodeSlowGuess tries to get a lexer by file name and language first,
// if not found, it will try to guess the lexer by code content, which is slow (more than several hundreds of milliseconds).
func RenderCodeSlowGuess(fileName, language, code string) (output template.HTML, lexer chroma.Lexer, lexerDisplayName string) {
// diff view newline will be passed as empty, change to literal '\n' so it can be copied
// preserve literal newline in blame view
if code == "" || code == "\n" {
return "\n", nil, ""
}
if len(code) > sizeLimit {
return htmlEscape(code), nil, ""
}
lexer = detectChromaLexerWithAnalyze(fileName, language, util.UnsafeStringToBytes(code)) // it is also slow
lexer = normalizeLexer(lexer)
return RenderCodeByLexer(lexer, code), lexer, formatLexerName(lexer.Config().Name)
}
// RenderCodeByLexer returns an HTML version of code string with chroma syntax highlighting classes
func RenderCodeByLexer(lexer chroma.Lexer, code string) template.HTML {
if lexer == lexers.Fallback {
return htmlEscape(code) // chroma is 3x slower for plain text
}
iterator, err := lexer.Tokenise(nil, code)
if err != nil {
log.Error("Can't tokenize code: %v", err)
return htmlEscape(code)
}
htmlBuf := &bytes.Buffer{}
vars := globalVars()
// style not used for live site but need to pass something
err = vars.htmlFormatter.Format(htmlBuf, vars.githubStyles, iterator)
if err != nil {
log.Error("Can't format code: %v", err)
return htmlEscape(code)
}
return template.HTML(util.UnsafeBytesToString(htmlBuf.Bytes()))
}
// RenderFullFile returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name
func RenderFullFile(fileName, language string, code []byte) ([]template.HTML, string) {
if language == LanguagePlaintext || len(code) > sizeLimit {
return renderPlainText(code), formatLexerName(LanguagePlaintext)
}
lexer := detectChromaLexerWithAnalyze(fileName, language, code)
lexer = normalizeLexer(lexer)
lexerName := formatLexerName(lexer.Config().Name)
rendered := RenderCodeByLexer(lexer, util.UnsafeBytesToString(code))
unsafeLines := UnsafeSplitHighlightedLines(rendered)
lines := make([]template.HTML, len(unsafeLines))
for idx, lineBytes := range unsafeLines {
lines[idx] = template.HTML(util.UnsafeBytesToString(lineBytes))
}
return lines, lexerName
}
// renderPlainText returns non-highlighted HTML for code
func renderPlainText(code []byte) []template.HTML {
lines := make([]template.HTML, 0, bytes.Count(code, []byte{'\n'})+1)
pos := 0
for pos < len(code) {
var content []byte
nextPos := bytes.IndexByte(code[pos:], '\n')
if nextPos == -1 {
content = code[pos:]
pos = len(code)
} else {
content = code[pos : pos+nextPos+1]
pos += nextPos + 1
}
lines = append(lines, htmlEscape(util.UnsafeBytesToString(content)))
}
return lines
}
func formatLexerName(name string) string {
if name == LanguagePlaintext || name == chromaLexerFallback {
return "Plaintext"
}
return util.ToTitleCaseNoLower(name)
}
func languageForCssAttrName(lang string) (forCSS, forAttr string) {
s := strings.ToLower(lang)
if s == "" || s == LanguagePlaintext || s == chromaLexerFallback {
return "text", "text"
}
isValid := func(c byte) bool {
// although "-" is valid in CSS name, it is used as a field separator, so we don't want to keep it in the name
return 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_'
}
idx := 0
for ; idx < len(s); idx++ {
if !isValid(s[idx]) {
break
}
}
if idx == len(s) {
return s, lang
}
out := []byte(s)
for i := idx; i < len(s); i++ {
if !isValid(out[i]) {
out[i] = '_'
}
}
return string(out), lang
}
func CodeBlockAttributes(lang string) (preAttrs, codeAttrs template.HTML) {
// Code block's "chroma" class is used to highlight the code.
// "language-{LanguageName}" class is used as part of commonmark spec.
// It's unclear about how to handle special chars for a language name like "Visual Basic.NET" or "C++" or "F#".
// The commonmark spec seems wrong: https://spec.commonmark.org/0.31.2/#info-string, it just outputs invalid CSS class names.
cssName, attrLang := languageForCssAttrName(lang)
renderByFrontend := lang == "mermaid" || lang == "math"
preExtraClasses := ""
if renderByFrontend {
preExtraClasses = " is-loading"
}
// The "math.ts" strictly depends on the structure: <pre class="code-block"><code class="language-math">...</code></pre>
// * If "pre" exists, it is rendered as "block", otherwise, it is rendered as "inline"
// The "mermaid.ts" also strictly depends on the structure: "pre" must exist because it is always rendered as "block".
//
// Hint: "data-code-language" is not exposed in some cases due to the Markup sanitizer, the rules can be refactored in the future if the attribute is useful.
return htmlutil.HTMLFormat(`class="code-block%s"`, preExtraClasses), htmlutil.HTMLFormat(`class="chroma language-%s" data-code-language="%s"`, cssName, attrLang)
}