make links in bluesky posts clickable

Bluesky linkifies nothing by itself. A URL in the text of a record is
plain text unless a richtext facet says which bytes of the post are a
link and where they point, so our posts carried URLs no one could press.

Each link now travels on the Post as an output.Link with the offsets of
the text it occupies, and the bluesky output turns those into
app.bsky.richtext.facet#link. The offsets are UTF-8 byte offsets, not
character counts: a facet measured in characters slides off the URL as
soon as any Japanese text precedes it, and underlines the wrong words.

Scanning moved from preview.go to findLinks in main.go, which now
reports every link with its offsets rather than just the first one; the
preview card still goes to the first, which is the one a reader meets
first. X is unaffected, as it linkifies URLs itself.

The facet feature only gains its lexicon type when marshalled, so the
test checks the encoded record rather than the struct.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/bluesky-social/indigo v0.0.0-20250313000755-d9a74f690c90
 	github.com/bwmarrin/discordgo v0.28.1
 	golang.org/x/image v0.43.0
+	golang.org/x/net v0.23.0
 )
 
 require (
@@ -51,6 +52,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/text v0.38.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	lukechampine.com/blake3 v1.2.1 // indirect
 )

go.sum

@@ -178,6 +178,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -196,6 +198,8 @@ golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

main.go

@@ -8,7 +8,9 @@ import (
 	"image/jpeg"
 	_ "image/png"
 	"io"
+	"mime"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -44,6 +46,24 @@ func trimURL(match string) string {
 	return strings.TrimRight(match, ".,!?、。)]}>")
 }
 
+// findLinks returns every link in content, each with the UTF-8 byte offsets
+// of the text it occupies.
+func findLinks(content string) []output.Link {
+	var links []output.Link
+	for _, span := range urlPattern.FindAllStringIndex(content, -1) {
+		raw := trimURL(content[span[0]:span[1]])
+		if u, err := url.Parse(raw); err != nil || u.Host == "" {
+			continue
+		}
+		links = append(links, output.Link{
+			URL:       raw,
+			ByteStart: span[0],
+			ByteEnd:   span[0] + len(raw),
+		})
+	}
+	return links
+}
+
 // tweetLength counts a message the way Twitter does, charging every link a
 // fixed length instead of its actual one.
 func tweetLength(content string) int {
@@ -55,8 +75,31 @@ func tweetLength(content string) int {
 	return length
 }
 
+// maxDownloadBytes caps an image download. Discord's own attachment limit is
+// well below it, so a truncated image means something else served us a body
+// far larger than any picture we would want to post.
+const maxDownloadBytes = 32 << 20
+
+// userAgent names the bot to the sites whose preview tags it reads; some of
+// them serve those tags only to a client that identifies itself.
+const userAgent = "tweetdistributor/1.0 (link preview)"
+
 var httpClient = &http.Client{Timeout: 30 * time.Second}
 
+// fetched is a downloaded document together with what the response said
+// about it.
+type fetched struct {
+	body []byte
+	// mediaType is the Content-Type without its parameters, e.g. "text/html".
+	mediaType string
+	// contentType is the header as sent, parameters and all, which is what
+	// tells a decoder the character encoding.
+	contentType string
+	// url is where the body actually came from, after any redirects, and is
+	// what relative links in it resolve against.
+	url *url.URL
+}
+
 // shrinkImage re-encodes (and if necessary downscales) an image until it
 // fits within maxImageBytes. Images already small enough pass through
 // untouched.
@@ -98,32 +141,52 @@ func shrinkImage(img output.Image) (output.Image, error) {
 	return output.Image{}, fmt.Errorf("%s could not be shrunk below %d bytes", img.Filename, maxImageBytes)
 }
 
-// fetch GETs url and returns its body.
-func fetch(url string) ([]byte, error) {
-	resp, err := httpClient.Get(url)
+// fetch GETs rawurl, reading at most limit bytes of the body. Callers that
+// only need the beginning of a document pass a small limit and treat the
+// truncation as normal.
+func fetch(rawurl string, limit int64) (*fetched, error) {
+	req, err := http.NewRequest(http.MethodGet, rawurl, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	data, err := io.ReadAll(resp.Body)
+	data, err := io.ReadAll(io.LimitReader(resp.Body, limit))
 	if err != nil {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("status %s", resp.Status)
 	}
-	return data, nil
+
+	contenttype := resp.Header.Get("Content-Type")
+	mediatype, _, err := mime.ParseMediaType(contenttype)
+	if err != nil {
+		mediatype = ""
+	}
+
+	return &fetched{
+		body:        data,
+		mediaType:   mediatype,
+		contentType: contenttype,
+		url:         resp.Request.URL,
+	}, nil
 }
 
 // downloadImage fetches an image and shrinks it to a postable size.
 func downloadImage(url, filename, contentType string) (output.Image, error) {
-	data, err := fetch(url)
+	got, err := fetch(url, maxDownloadBytes)
 	if err != nil {
 		return output.Image{}, fmt.Errorf("downloading %s: %w", filename, err)
 	}
 	return shrinkImage(output.Image{
-		Data:        data,
+		Data:        got.body,
 		ContentType: contentType,
 		Filename:    filename,
 	})
@@ -176,9 +239,11 @@ func (dist *distributor) created(event discord.Event) {
 		return
 	}
 
+	// The card goes to the first link, the one a reader meets first.
+	links := findLinks(event.Content)
 	var preview *output.Preview
-	if videoURL := findYouTubeURL(event.Content); videoURL != "" {
-		preview, err = youtubePreview(videoURL)
+	if len(links) > 0 {
+		preview, err = linkPreview(links[0].URL)
 		if err != nil {
 			// The post is still worth making without its card.
 			fmt.Fprintln(os.Stderr, err)
@@ -196,6 +261,7 @@ func (dist *distributor) created(event discord.Event) {
 		post := output.Post{
 			Text:    event.Content,
 			Images:  images,
+			Links:   links,
 			Preview: preview,
 		}
 		if parent, ok := parents[out.GetName()]; ok && !parent.IsZero() {

main_test.go

@@ -30,6 +30,55 @@ func noiseJPEG(t *testing.T, width, height int) []byte {
 	return buf.Bytes()
 }
 
+func TestFindLinks(t *testing.T) {
+	tests := []struct {
+		name    string
+		content string
+		want    []output.Link
+	}{
+		{
+			// The offsets are byte offsets, so the multibyte prefix counts
+			// for more than the three characters it looks like.
+			"after multibyte text",
+			"みてね https://example.com/article",
+			[]output.Link{{URL: "https://example.com/article", ByteStart: 10, ByteEnd: 37}},
+		},
+		{
+			"trailing punctuation is outside the link",
+			"これ→https://example.com/x。",
+			[]output.Link{{URL: "https://example.com/x", ByteStart: 9, ByteEnd: 30}},
+		},
+		{
+			"several",
+			"https://one.example https://two.example",
+			[]output.Link{
+				{URL: "https://one.example", ByteStart: 0, ByteEnd: 19},
+				{URL: "https://two.example", ByteStart: 20, ByteEnd: 39},
+			},
+		},
+		{"no link", "ただのつぶやき", nil},
+		{"scheme only", "http:// と書いただけ", nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := findLinks(tt.content)
+			if len(got) != len(tt.want) {
+				t.Fatalf("findLinks(%q) returned %d links, want %d", tt.content, len(got), len(tt.want))
+			}
+			for i, link := range got {
+				if link != tt.want[i] {
+					t.Errorf("link %d = %+v, want %+v", i, link, tt.want[i])
+				}
+				// A facet pointing at the wrong bytes marks up the wrong text.
+				if slice := tt.content[link.ByteStart:link.ByteEnd]; slice != link.URL {
+					t.Errorf("bytes [%d:%d] are %q, want %q", link.ByteStart, link.ByteEnd, slice, link.URL)
+				}
+			}
+		})
+	}
+}
+
 func TestTweetLength(t *testing.T) {
 	tests := []struct {
 		name    string

output/bluesky.go

@@ -67,6 +67,8 @@ func (bo *blueskyoutput) Write(post Post) (Ref, error) {
 		Langs:     []string{"ja"},
 	}
 
+	feedpost.Facets = linkFacets(post.Links)
+
 	root := Ref{}
 	if post.ReplyTo != nil && post.ReplyTo.URI != "" {
 		parent := &atproto.RepoStrongRef{
@@ -146,6 +148,29 @@ func (bo *blueskyoutput) Write(post Post) (Ref, error) {
 	}, nil
 }
 
+// linkFacets marks up the links in a post's text. Bluesky linkifies nothing
+// by itself: a URL stays plain text until a facet says which bytes of the
+// post are a link and where they point.
+func linkFacets(links []Link) []*bsky.RichtextFacet {
+	if len(links) == 0 {
+		return nil
+	}
+
+	facets := make([]*bsky.RichtextFacet, 0, len(links))
+	for _, link := range links {
+		facets = append(facets, &bsky.RichtextFacet{
+			Index: &bsky.RichtextFacet_ByteSlice{
+				ByteStart: int64(link.ByteStart),
+				ByteEnd:   int64(link.ByteEnd),
+			},
+			Features: []*bsky.RichtextFacet_Features_Elem{{
+				RichtextFacet_Link: &bsky.RichtextFacet_Link{Uri: link.URL},
+			}},
+		})
+	}
+	return facets
+}
+
 func (bo *blueskyoutput) Delete(ref Ref) error {
 	rkey, err := recordKey(ref.URI)
 	if err != nil {

output/bluesky_test.go

@@ -1,6 +1,51 @@
 package output
 
-import "testing"
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/util"
+)
+
+func TestLinkFacets(t *testing.T) {
+	if facets := linkFacets(nil); facets != nil {
+		t.Errorf("a post with no links got %d facets, want none", len(facets))
+	}
+
+	text := "みてね https://example.com/x"
+	post := &bsky.FeedPost{
+		Text:      text,
+		CreatedAt: time.Now().Format(util.ISO8601),
+		Facets:    linkFacets([]Link{{URL: "https://example.com/x", ByteStart: 10, ByteEnd: 31}}),
+	}
+
+	// The feature carries its lexicon type only once marshalled, so the
+	// record as it goes over the wire is what has to be checked.
+	encoded, err := json.Marshal(post)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, want := range []string{
+		`"$type":"app.bsky.richtext.facet#link"`,
+		`"uri":"https://example.com/x"`,
+		`"byteStart":10`,
+		`"byteEnd":31`,
+	} {
+		if !strings.Contains(string(encoded), want) {
+			t.Errorf("record does not contain %s\ngot: %s", want, encoded)
+		}
+	}
+
+	// A facet that points at the wrong bytes underlines the wrong text.
+	index := post.Facets[0].Index
+	if slice := text[index.ByteStart:index.ByteEnd]; slice != "https://example.com/x" {
+		t.Errorf("facet covers %q, want the URL", slice)
+	}
+}
 
 func TestRecordKey(t *testing.T) {
 	rkey, err := recordKey("at://did:plc:abc123/app.bsky.feed.post/3kqz7xyz")

output/output.go

@@ -14,6 +14,15 @@ type Preview struct {
 	Thumb       *Image
 }
 
+// Link is a URL inside a post's text, located by the UTF-8 byte offsets
+// bluesky needs to mark it up: it does not linkify text on its own, so a URL
+// no one points at stays unclickable.
+type Link struct {
+	URL       string
+	ByteStart int
+	ByteEnd   int
+}
+
 // Ref identifies a post an output has already published so it can later be
 // replied to or deleted. The fields are output specific; only the ones the
 // publishing output filled in are meaningful to it.
@@ -34,6 +43,7 @@ func (r Ref) IsZero() bool {
 type Post struct {
 	Text    string
 	Images  []Image
+	Links   []Link
 	Preview *Preview
 	// ReplyTo is the Ref this same output returned for the post being
 	// replied to, or nil for a top level post.

preview.go

@@ -1,37 +1,49 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"os"
+	"path"
 	"strings"
 	"tweetdistributor/output"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/charset"
 )
 
-// findYouTubeURL returns the first YouTube video link in content, or "" if
-// there is none.
-func findYouTubeURL(content string) string {
-	for _, match := range urlPattern.FindAllString(content, -1) {
-		raw := trimURL(match)
-		u, err := url.Parse(raw)
+// maxHTMLBytes caps how much of a page is read while looking for its preview
+// tags. They belong in <head>, so reading further is wasted.
+const maxHTMLBytes = 1 << 20
+
+// linkPreview builds the preview card for a link. Bluesky shows no card of
+// its own: whatever the record does not embed is not displayed, so every card
+// has to be assembled here.
+func linkPreview(link string) (*output.Preview, error) {
+	u, err := url.Parse(link)
 	if err != nil {
-			continue
+		return nil, fmt.Errorf("parsing %s: %w", link, err)
 	}
+	if isYouTube(u) {
+		return youtubePreview(u)
+	}
+	return ogpPreview(u)
+}
+
+// isYouTube reports whether u addresses a YouTube video.
+func isYouTube(u *url.URL) bool {
 	switch strings.ToLower(u.Hostname()) {
 	case "youtu.be":
-			if strings.Trim(u.Path, "/") != "" {
-				return raw
-			}
+		return strings.Trim(u.Path, "/") != ""
 	case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
 		if u.Path == "/watch" && u.Query().Get("v") != "" {
-				return raw
+			return true
 		}
-			if strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/") {
-				return raw
+		return strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/")
 	}
-		}
-	}
-	return ""
+	return false
 }
 
 // oEmbedResponse is the part of YouTube's oEmbed document we care about.
@@ -41,34 +53,161 @@ type oEmbedResponse struct {
 	ThumbnailURL string `json:"thumbnail_url"`
 }
 
-// youtubePreview builds the preview card for a YouTube link by asking
-// YouTube's oEmbed endpoint for the title, channel and thumbnail.
-func youtubePreview(videoURL string) (*output.Preview, error) {
-	endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(videoURL)
-	body, err := fetch(endpoint)
+// youtubePreview builds the card for a YouTube link from the oEmbed endpoint,
+// which answers with just the few fields a card needs instead of the megabyte
+// of markup the watch page is.
+func youtubePreview(video *url.URL) (*output.Preview, error) {
+	endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(video.String())
+	got, err := fetch(endpoint, maxHTMLBytes)
 	if err != nil {
-		return nil, fmt.Errorf("fetching preview for %s: %w", videoURL, err)
+		return nil, fmt.Errorf("fetching preview for %s: %w", video, err)
 	}
 
 	var oembed oEmbedResponse
-	if err := json.Unmarshal(body, &oembed); err != nil {
-		return nil, fmt.Errorf("parsing preview for %s: %w", videoURL, err)
+	if err := json.Unmarshal(got.body, &oembed); err != nil {
+		return nil, fmt.Errorf("parsing preview for %s: %w", video, err)
 	}
 
 	preview := &output.Preview{
-		URL:         videoURL,
+		URL:         video.String(),
 		Title:       oembed.Title,
 		Description: oembed.AuthorName,
 	}
-
 	if oembed.ThumbnailURL != "" {
-		thumb, err := downloadImage(oembed.ThumbnailURL, "thumbnail.jpg", "image/jpeg")
-		if err != nil {
-			// A card without its thumbnail is still worth posting.
-			return preview, nil
+		preview.Thumb = thumbnail(video, oembed.ThumbnailURL)
 	}
-		preview.Thumb = &thumb
+	return preview, nil
 }
 
+// ogpPreview builds the card for an ordinary page from its Open Graph tags,
+// falling back to the Twitter card tags and then to the plain document title.
+func ogpPreview(page *url.URL) (*output.Preview, error) {
+	got, err := fetch(page.String(), maxHTMLBytes)
+	if err != nil {
+		return nil, fmt.Errorf("fetching preview for %s: %w", page, err)
+	}
+	switch got.mediaType {
+	case "", "text/html", "application/xhtml+xml":
+	default:
+		return nil, fmt.Errorf("%s is %s, which carries no preview tags", page, got.mediaType)
+	}
+
+	tags, err := parseMetaTags(got)
+	if err != nil {
+		return nil, fmt.Errorf("reading preview for %s: %w", page, err)
+	}
+
+	title := tags.first("og:title", "twitter:title", "title")
+	if title == "" {
+		return nil, fmt.Errorf("%s has no title to put on a card", page)
+	}
+
+	preview := &output.Preview{
+		// The card links to the page as it was written, not as it redirected.
+		URL:         page.String(),
+		Title:       title,
+		Description: tags.first("og:description", "twitter:description", "description"),
+	}
+	if image := tags.first("og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src"); image != "" {
+		preview.Thumb = thumbnail(got.url, image)
+	}
 	return preview, nil
 }
+
+// metaTags holds a page's <meta> tags keyed by their property or name
+// attribute, plus its <title> under "title".
+type metaTags map[string]string
+
+// first returns the value of the earliest of keys that the page set.
+func (tags metaTags) first(keys ...string) string {
+	for _, key := range keys {
+		if value := strings.TrimSpace(tags[key]); value != "" {
+			return value
+		}
+	}
+	return ""
+}
+
+// parseMetaTags reads the tags out of a page. It stops at <body>, past which
+// preview tags do not belong, and treats running out of markup as the end:
+// the body was cut off at maxHTMLBytes.
+func parseMetaTags(page *fetched) (metaTags, error) {
+	// Pages are not all UTF-8; charset works out the encoding from the
+	// Content-Type header, a byte order mark or the meta charset tag.
+	decoded, err := charset.NewReader(bytes.NewReader(page.body), page.contentType)
+	if err != nil {
+		return nil, err
+	}
+
+	tags := metaTags{}
+	tokenizer := html.NewTokenizer(decoded)
+	for {
+		switch tokenizer.Next() {
+		case html.ErrorToken:
+			return tags, nil
+		case html.StartTagToken, html.SelfClosingTagToken:
+			name, hasattr := tokenizer.TagName()
+			switch string(name) {
+			case "meta":
+				if !hasattr {
+					continue
+				}
+				var key, content string
+				for {
+					attr, value, more := tokenizer.TagAttr()
+					switch string(attr) {
+					case "property", "name":
+						key = strings.ToLower(string(value))
+					case "content":
+						content = string(value)
+					}
+					if !more {
+						break
+					}
+				}
+				// The first tag of a name wins, as it does in every reader.
+				if key != "" && content != "" && tags[key] == "" {
+					tags[key] = content
+				}
+			case "title":
+				if tokenizer.Next() == html.TextToken && tags["title"] == "" {
+					tags["title"] = strings.TrimSpace(string(tokenizer.Text()))
+				}
+			case "body":
+				return tags, nil
+			}
+		}
+	}
+}
+
+// thumbnail fetches a card image, resolving ref against the page it was found
+// on. A card without its picture is still worth posting, so a thumbnail that
+// cannot be fetched is reported and dropped rather than failing the card.
+func thumbnail(base *url.URL, ref string) *output.Image {
+	imageurl, err := base.Parse(ref)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", ref, err)
+		return nil
+	}
+
+	got, err := fetch(imageurl.String(), maxDownloadBytes)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
+		return nil
+	}
+
+	filename := path.Base(imageurl.Path)
+	if filename == "." || filename == "/" {
+		filename = "thumbnail"
+	}
+	img, err := shrinkImage(output.Image{
+		Data:        got.body,
+		ContentType: got.mediaType,
+		Filename:    filename,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
+		return nil
+	}
+	return &img
+}

preview_test.go

@@ -1,33 +1,110 @@
 package main
 
-import "testing"
+import (
+	"net/url"
+	"testing"
+)
 
-func TestFindYouTubeURL(t *testing.T) {
+func TestIsYouTube(t *testing.T) {
 	tests := []struct {
-		name    string
-		content string
-		want    string
+		raw  string
+		want bool
 	}{
-		{"watch", "みてみて https://www.youtube.com/watch?v=dQw4w9WgXcQ おもしろい", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
-		{"short host", "https://youtu.be/dQw4w9WgXcQ?t=42", "https://youtu.be/dQw4w9WgXcQ?t=42"},
-		{"shorts", "https://www.youtube.com/shorts/abc_123", "https://www.youtube.com/shorts/abc_123"},
-		{"live", "https://youtube.com/live/abc-123", "https://youtube.com/live/abc-123"},
-		{"mobile", "https://m.youtube.com/watch?v=abc&feature=share", "https://m.youtube.com/watch?v=abc&feature=share"},
-		{"trailing punctuation", "これ→https://youtu.be/abc123。", "https://youtu.be/abc123"},
-		{"first of several", "https://youtu.be/one https://youtu.be/two", "https://youtu.be/one"},
-		{"skips other links", "https://example.com/watch?v=x https://youtu.be/abc", "https://youtu.be/abc"},
-		{"no url", "ただのつぶやき", ""},
-		{"other site", "https://example.com/", ""},
-		{"youtube without video", "https://www.youtube.com/", ""},
-		{"watch without v", "https://www.youtube.com/watch?list=PL123", ""},
-		{"lookalike host", "https://youtube.com.evil.example/watch?v=abc", ""},
+		{"https://www.youtube.com/watch?v=dQw4w9WgXcQ", true},
+		{"https://youtu.be/dQw4w9WgXcQ?t=42", true},
+		{"https://www.youtube.com/shorts/abc_123", true},
+		{"https://youtube.com/live/abc-123", true},
+		{"https://m.youtube.com/watch?v=abc&feature=share", true},
+		{"https://www.youtube.com/", false},
+		{"https://www.youtube.com/watch?list=PL123", false},
+		{"https://youtube.com.evil.example/watch?v=abc", false},
+		{"https://example.com/", false},
 	}
 
 	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := findYouTubeURL(tt.content); got != tt.want {
-				t.Errorf("findYouTubeURL(%q) = %q, want %q", tt.content, got, tt.want)
+		t.Run(tt.raw, func(t *testing.T) {
+			u, err := url.Parse(tt.raw)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := isYouTube(u); got != tt.want {
+				t.Errorf("isYouTube(%q) = %v, want %v", tt.raw, got, tt.want)
 			}
 		})
 	}
 }
+
+func TestParseMetaTags(t *testing.T) {
+	page := &fetched{
+		contentType: "text/html; charset=utf-8",
+		body: []byte(`<!doctype html><html><head>
+			<title>plain title</title>
+			<meta name="description" content="plain description">
+			<meta property="og:title" content="OGP タイトル">
+			<meta property="og:description" content="OGP の説明 &amp; その続き" />
+			<meta content="https://example.com/card.png" property="og:image">
+			<meta property="og:title" content="a later duplicate">
+		</head><body>
+			<meta property="og:image" content="https://example.com/inbody.png">
+		</body></html>`),
+	}
+
+	tags, err := parseMetaTags(page)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := map[string]string{
+		"title":          "plain title",
+		"description":    "plain description",
+		"og:title":       "OGP タイトル",
+		"og:description": "OGP の説明 & その続き",
+		"og:image":       "https://example.com/card.png",
+	}
+	for key, value := range want {
+		if tags[key] != value {
+			t.Errorf("tags[%q] = %q, want %q", key, tags[key], value)
+		}
+	}
+
+	if got := tags.first("og:title", "twitter:title", "title"); got != "OGP タイトル" {
+		t.Errorf("first title = %q, want the OGP one", got)
+	}
+	if got := tags.first("twitter:title", "title"); got != "plain title" {
+		t.Errorf("first title = %q, want the fallback", got)
+	}
+	if got := tags.first("nothing:here"); got != "" {
+		t.Errorf("first of an absent key = %q, want empty", got)
+	}
+}
+
+func TestParseMetaTagsShiftJIS(t *testing.T) {
+	// "テスト" encoded as Shift_JIS, declared in the Content-Type header.
+	body := []byte(`<html><head><meta property="og:title" content="` +
+		"\x83e\x83X\x83g" + `"></head></html>`)
+	page := &fetched{contentType: "text/html; charset=Shift_JIS", body: body}
+
+	tags, err := parseMetaTags(page)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tags["og:title"] != "テスト" {
+		t.Errorf("og:title = %q, want テスト", tags["og:title"])
+	}
+}
+
+func TestParseMetaTagsTruncated(t *testing.T) {
+	// A page cut off at maxHTMLBytes ends mid markup; what was read still counts.
+	page := &fetched{
+		contentType: "text/html",
+		body:        []byte(`<html><head><meta property="og:title" content="kept"><meta property="og:desc`),
+	}
+
+	tags, err := parseMetaTags(page)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tags["og:title"] != "kept" {
+		t.Errorf("og:title = %q, want kept", tags["og:title"])
+	}
+}