From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Web Agent Date: Fri, 9 Oct 2026 13:33:02 +0200 Subject: [PATCH] perf: scan e-mail domains and linkify's URLs without regexps util.FindEmailIndex scans the domain as goldmark v2's parser does (scanEmailDomain, ea86b6d5), and linkify scans URLs and www. links with scanURLHost, which finds what its patterns found: the host is the longest run of up to 256 host characters that a '.' and a lowercase letter follow, and the rest of the pattern is optional. WithLinkifyURLRegexp and WithLinkifyWWWRegexp still take a pattern; only the defaults are gone. The patterns were compiled at package initialization, by every program that imports util (all of goldmark) or extension. Their {0,61} and {1,256} repeats compile into regexps nested hundreds deep, which overflows the fixed 64 KB goroutine stacks of TinyGo's WebAssembly targets: goldmark crashed such programs before main (tinygo-org/tinygo#3258). TestLinkifyScannersMatchRegexps checks the scanners against the old patterns on 300,000 generated lines. --- extension/linkify.go | 115 +++++++++++++++++++++++++++++--- extension/linkify_bench_test.go | 25 +++++++ extension/linkify_scan_test.go | 98 +++++++++++++++++++++++++++ util/util.go | 43 ++++++++++-- 4 files changed, 266 insertions(+), 15 deletions(-) create mode 100644 extension/linkify_bench_test.go create mode 100644 extension/linkify_scan_test.go diff --git a/extension/linkify.go b/extension/linkify.go index ca17247..c15c1c5 100644 --- a/extension/linkify.go +++ b/extension/linkify.go @@ -11,10 +11,6 @@ import ( "github.com/yuin/goldmark/util" ) -var wwwURLRegxp = regexp.MustCompile(`^www\.[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?:[/#?][-a-zA-Z0-9@:%_\+.~#!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) //nolint:golint,lll - -var urlRegexp = regexp.MustCompile(`^(?:http|https|ftp)://[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?::\d+)?(?:[/#?][-a-zA-Z0-9@:%_+.~#$!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) //nolint:golint,lll - // An LinkifyConfig struct is a data structure that holds configuration of the // Linkify extension. type LinkifyConfig struct { @@ -145,8 +141,6 @@ func NewLinkifyParser(opts ...LinkifyOption) parser.InlineParser { p := &linkifyParser{ LinkifyConfig: LinkifyConfig{ AllowedProtocols: nil, - URLRegexp: urlRegexp, - WWWRegexp: wwwURLRegxp, }, } for _, o := range opts { @@ -187,18 +181,18 @@ func (s *linkifyParser) Parse(parent ast.Node, block text.Reader, pc parser.Cont var typ ast.AutoLinkType = ast.AutoLinkURL if s.LinkifyConfig.AllowedProtocols == nil { if bytes.HasPrefix(line, protoHTTP) || bytes.HasPrefix(line, protoHTTPS) || bytes.HasPrefix(line, protoFTP) { - m = s.LinkifyConfig.URLRegexp.FindSubmatchIndex(line) + m = s.findURL(line) } } else { for _, prefix := range s.LinkifyConfig.AllowedProtocols { if bytes.HasPrefix(line, prefix) { - m = s.LinkifyConfig.URLRegexp.FindSubmatchIndex(line) + m = s.findURL(line) break } } } if m == nil && bytes.HasPrefix(line, domainWWW) { - m = s.LinkifyConfig.WWWRegexp.FindSubmatchIndex(line) + m = s.findWWW(line) protocol = []byte("http") } if m != nil && m[0] != 0 { @@ -303,3 +297,106 @@ func (e *linkify) Extend(m goldmark.Markdown) { ), ) } + +var urlSchemes = [][]byte{[]byte("http://"), []byte("https://"), []byte("ftp://")} + +// findURL returns the bounds of the URL with a protocol at the start of +// line, or nil. +func (s *linkifyParser) findURL(line []byte) []int { + if s.LinkifyConfig.URLRegexp != nil { + return s.LinkifyConfig.URLRegexp.FindSubmatchIndex(line) + } + for _, scheme := range urlSchemes { + if bytes.HasPrefix(line, scheme) { + if n := scanURLHost(line, len(scheme), true); n > 0 { + return []int{0, n} + } + return nil + } + } + return nil +} + +// findWWW returns the bounds of the URL starting with "www." at the start +// of line, or nil. +func (s *linkifyParser) findWWW(line []byte) []int { + if s.LinkifyConfig.WWWRegexp != nil { + return s.LinkifyConfig.WWWRegexp.FindSubmatchIndex(line) + } + if !bytes.HasPrefix(line, domainWWW) { + return nil + } + if n := scanURLHost(line, len(domainWWW), false); n > 0 { + return []int{0, n} + } + return nil +} + +const ( + urlHost = 1 << iota // [-a-zA-Z0-9@:%._+~#=] + urlPath // [-a-zA-Z0-9@:%_+.~#!?&/=();,'">^{}[\]`] + urlPathWithProtocol // urlPath and '$' +) + +var urlChars = func() (t [256]uint8) { + for c := 0; c < 256; c++ { + if util.IsAlphaNumeric(byte(c)) { + t[c] = urlHost | urlPath | urlPathWithProtocol + } + } + for _, c := range []byte("-@:%._+~#=") { + t[c] |= urlHost + } + for _, c := range []byte("-@:%_+.~#!?&/=();,'\">^{}[]`") { + t[c] |= urlPath | urlPathWithProtocol + } + t['$'] |= urlPathWithProtocol + return t +}() + +func isLowerAlpha(c byte) bool { + return 'a' <= c && c <= 'z' +} + +// scanURLHost returns where a URL ends whose scheme ends at i, or 0. It +// matches what +// +// [-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-z]+(?::\d+)?(?:[/#?][-a-zA-Z0-9@:%_+.~#$!?&/=();,'">^{}[\]`]*)? +// +// does from i: the host is the longest run of up to 256 host characters +// that a '.' and a lowercase letter follow. withProtocol allows a port, and +// a '$' in the path; without it, as after "www.", neither. +func scanURLHost(b []byte, i int, withProtocol bool) int { + j := i + for j < len(b) && j-i < 256 && urlChars[b[j]]&urlHost != 0 { + j++ + } + k := j + for k > i && !(k+1 < len(b) && b[k] == '.' && isLowerAlpha(b[k+1])) { + k-- + } + if k == i { + return 0 + } + n := k + 2 + for n < len(b) && isLowerAlpha(b[n]) { + n++ + } + if withProtocol && n+1 < len(b) && b[n] == ':' && util.IsNumeric(b[n+1]) { + n += 2 + for n < len(b) && util.IsNumeric(b[n]) { + n++ + } + } + if n < len(b) && (b[n] == '/' || b[n] == '#' || b[n] == '?') { + path := uint8(urlPath) + if withProtocol { + path = urlPathWithProtocol + } + n++ + for n < len(b) && urlChars[b[n]]&path != 0 { + n++ + } + } + return n +} diff --git a/extension/linkify_bench_test.go b/extension/linkify_bench_test.go new file mode 100644 index 0000000..4119496 --- /dev/null +++ b/extension/linkify_bench_test.go @@ -0,0 +1,25 @@ +package extension + +import ( + "bytes" + "strings" + "testing" + + "github.com/yuin/goldmark" +) + +func BenchmarkLinkify(b *testing.B) { + line := "See www.example.org/docs, https://example.com:8080/a/b?c=d#e and mail me at someone@mail.example.net (or ftp://files.example.org/pub).\n" + src := []byte(strings.Repeat(line, 200)) + md := goldmark.New(goldmark.WithExtensions(Linkify)) + var buf bytes.Buffer + b.SetBytes(int64(len(src))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + buf.Reset() + if err := md.Convert(src, &buf); err != nil { + b.Fatal(err) + } + } +} diff --git a/extension/linkify_scan_test.go b/extension/linkify_scan_test.go new file mode 100644 index 0000000..2511ec6 --- /dev/null +++ b/extension/linkify_scan_test.go @@ -0,0 +1,98 @@ +package extension + +import ( + "math/rand/v2" + "regexp" + "strings" + "testing" + + "github.com/yuin/goldmark/util" +) + +// The patterns linkify matched with before it scanned; the scanners must find +// what they found. +var ( + oracleWWW = regexp.MustCompile(`^www\.[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?:[/#?][-a-zA-Z0-9@:%_\+.~#!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) //nolint:lll + oracleURL = regexp.MustCompile(`^(?:http|https|ftp)://[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-z]+(?::\d+)?(?:[/#?][-a-zA-Z0-9@:%_+.~#$!?&/=\(\);,'">\^{}\[\]` + "`" + `]*)?`) //nolint:lll + oracleEmail = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*`) //nolint:lll +) + +var linkifyPieces = []string{ + "a", "z", "Q", "0", "9", ".", "-", "_", ":", "/", "#", "?", "$", "@", "%", "+", "~", "=", + "!", "&", "(", ")", ";", ",", "'", "\"", ">", "<", "^", "{", "}", "[", "]", "`", " ", "\\", + "\x80", "é", ":80", ".com", ".org", ".C", "..", "-.", ".-", "www.", "http://", +} + +func randomLinkifyLine(r *rand.Rand) string { + var b strings.Builder + switch r.IntN(4) { + case 0: + b.WriteString("http://") + case 1: + b.WriteString("https://") + case 2: + b.WriteString("www.") + } + for n := r.IntN(12); n > 0; n-- { + if r.IntN(8) == 0 { + // A run that ends around the 256 or the 63 limit. + c := []string{"a", "-", ".", "x.y", "Z"}[r.IntN(5)] + b.WriteString(strings.Repeat(c, []int{61, 62, 63, 64, 255, 256, 257, 300}[r.IntN(8)]/len(c))) + continue + } + b.WriteString(linkifyPieces[r.IntN(len(linkifyPieces))]) + } + return b.String() +} + +func oracleEnd(re *regexp.Regexp, s string) int { + m := re.FindStringSubmatchIndex(s) + if m == nil || m[0] != 0 { + return -1 + } + return m[1] +} + +func scannerEnd(m []int) int { + if m == nil || m[0] != 0 { + return -1 + } + return m[1] +} + +func TestLinkifyScannersMatchRegexps(t *testing.T) { + p := NewLinkifyParser().(*linkifyParser) + r := rand.New(rand.NewPCG(7, 279)) + edge := []string{ + "www.a.b", "www..b", "www.a.", "www.a.B", "www.a.b:80/x", "http://a.b:80/x$y", "www.a.b/x$y", + "http://a.b:x", "http://a.b:", "https://a.b?q=1&r=(2)", "ftp://x.y#z", "http://.a", "http://a..b", + "www." + strings.Repeat("a", 256) + ".com", "www." + strings.Repeat("a", 257) + ".com", + "www." + strings.Repeat("a.", 140) + "com", "http://" + strings.Repeat("a", 255) + ".b", + } + for i := 0; i < 300000; i++ { + s := randomLinkifyLine(r) + if i < len(edge) { + s = edge[i] + } + if got, want := scannerEnd(p.findURL([]byte(s))), oracleEnd(oracleURL, s); got != want { + t.Fatalf("url %q: scanned to %d, the regexp to %d", s, got, want) + } + if got, want := scannerEnd(p.findWWW([]byte(s))), oracleEnd(oracleWWW, s); got != want { + t.Fatalf("www %q: scanned to %d, the regexp to %d", s, got, want) + } + d := s + if j := strings.IndexByte(s, '/'); j >= 0 && r.IntN(2) == 0 { + d = s[j+1:] + } + want := oracleEnd(oracleEmail, d) + if want <= 0 { + want = -1 + } else { + want += len("a@") + } + got := util.FindEmailIndex([]byte("a@" + d)) + if got != want { + t.Fatalf("email a@%q: scanned to %d, the regexp to %d", d, got, want) + } + } +} diff --git a/util/util.go b/util/util.go index 2e8e188..79da271 100644 --- a/util/util.go +++ b/util/util.go @@ -5,7 +5,6 @@ import ( "bytes" "io" "net/url" - "regexp" "slices" "sort" "strconv" @@ -758,11 +757,43 @@ func FindURLIndex(b []byte) int { return i } -var emailDomainRegexp = regexp.MustCompile(`^[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*`) //nolint:golint,lll +// scanDomainLabel returns the length of the domain label at the start of b: +// a letter or digit, then up to 62 letters, digits and hyphens, ending in a +// letter or digit. It returns 0 if there is none. +func scanDomainLabel(b []byte) int { + if len(b) == 0 || !IsAlphaNumeric(b[0]) { + return 0 + } + limit := min(len(b), 63) // 1 + 61 + 1 + i := 1 + for i < limit && (IsAlphaNumeric(b[i]) || b[i] == '-') { + i++ + } + for i > 1 && b[i-1] == '-' { + i-- + } + return i +} + +// scanEmailDomain returns the length of the domain at the start of b: labels +// joined by dots. It returns 0 if there is none. +func scanEmailDomain(b []byte) int { + n := scanDomainLabel(b) + if n == 0 { + return 0 + } + for n < len(b) && b[n] == '.' { + m := scanDomainLabel(b[n+1:]) + if m == 0 { + break + } + n += 1 + m + } + return n +} // FindEmailIndex returns a stop index value if the given bytes seem an email address. func FindEmailIndex(b []byte) int { - // TODO: eliminate regexps i := 0 for ; i < len(b); i++ { c := b[i] @@ -780,11 +811,11 @@ func FindEmailIndex(b []byte) int { if i >= len(b) { return -1 } - match := emailDomainRegexp.FindSubmatchIndex(b[i:]) - if match == nil { + n := scanEmailDomain(b[i:]) + if n == 0 { return -1 } - return i + match[1] + return i + n } var spaces = []byte(" \t\n\x0b\x0c\x0d")