expand a preview card for any link, not just YouTube

Bluesky renders no link card of its own. Its PDS never fetches the page behind a URL, so a post whose record has no app.bsky.embed.external shows the link as bare text; every client that shows a card builds it before posting. We only built one for YouTube, so every other link arrived on bluesky with nothing attached. preview.go now reads the Open Graph tags of an ordinary page and turns them into the same card: og:title and og:description, falling back to the twitter:* tags and then to <title> and meta description, with og:image fetched as the thumbnail. YouTube keeps its oEmbed path, which answers with the handful of fields a card needs rather than the megabyte of markup the watch page is. Only the head of a page is read, and parsing stops at <body>, since preview tags belong above it and a truncated page still yields what was read. Pages are decoded through x/net/html/charset rather than assumed to be UTF-8, which Japanese pages served as Shift_JIS are not. The thumbnail is resolved against the URL the body came from, so a relative og:image survives a redirect. Requests now name the bot in a User-Agent, which some sites want before serving preview tags at all. fetch grew a byte limit and now reports the media type and final URL its body came with, which is what the thumbnail needs to name and resolve itself. Checked against go.dev, Japanese Wikipedia and a YouTube video. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-03 09:32:12 +09:00
parent e2f8036682
commit 953664ff1c
5 changed files with 363 additions and 62 deletions
--- a/main.go
+++ b/main.go
@@ -8,7 +8,9 @@ import (
 	"image/jpeg"
 	_ "image/png"
 	"io"
+	"mime"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -55,8 +57,31 @@ func tweetLength(content string) int {
 	return length
 }

+// maxDownloadBytes caps an image download. Discord's own attachment limit is
+// well below it, so a truncated image means something else served us a body
+// far larger than any picture we would want to post.
+const maxDownloadBytes = 32 << 20
+
+// userAgent names the bot to the sites whose preview tags it reads; some of
+// them serve those tags only to a client that identifies itself.
+const userAgent = "tweetdistributor/1.0 (link preview)"
+
 var httpClient = &http.Client{Timeout: 30 * time.Second}

+// fetched is a downloaded document together with what the response said
+// about it.
+type fetched struct {
+	body []byte
+	// mediaType is the Content-Type without its parameters, e.g. "text/html".
+	mediaType string
+	// contentType is the header as sent, parameters and all, which is what
+	// tells a decoder the character encoding.
+	contentType string
+	// url is where the body actually came from, after any redirects, and is
+	// what relative links in it resolve against.
+	url *url.URL
+}
+
 // shrinkImage re-encodes (and if necessary downscales) an image until it
 // fits within maxImageBytes. Images already small enough pass through
 // untouched.
@@ -98,32 +123,52 @@ func shrinkImage(img output.Image) (output.Image, error) {
 	return output.Image{}, fmt.Errorf("%s could not be shrunk below %d bytes", img.Filename, maxImageBytes)
 }

-// fetch GETs url and returns its body.
-func fetch(url string) ([]byte, error) {
-	resp, err := httpClient.Get(url)
+// fetch GETs rawurl, reading at most limit bytes of the body. Callers that
+// only need the beginning of a document pass a small limit and treat the
+// truncation as normal.
+func fetch(rawurl string, limit int64) (*fetched, error) {
+	req, err := http.NewRequest(http.MethodGet, rawurl, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()

-	data, err := io.ReadAll(resp.Body)
+	data, err := io.ReadAll(io.LimitReader(resp.Body, limit))
 	if err != nil {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("status %s", resp.Status)
 	}
-	return data, nil
+
+	contenttype := resp.Header.Get("Content-Type")
+	mediatype, _, err := mime.ParseMediaType(contenttype)
+	if err != nil {
+		mediatype = ""
+	}
+
+	return &fetched{
+		body:        data,
+		mediaType:   mediatype,
+		contentType: contenttype,
+		url:         resp.Request.URL,
+	}, nil
 }

 // downloadImage fetches an image and shrinks it to a postable size.
 func downloadImage(url, filename, contentType string) (output.Image, error) {
-	data, err := fetch(url)
+	got, err := fetch(url, maxDownloadBytes)
 	if err != nil {
 		return output.Image{}, fmt.Errorf("downloading %s: %w", filename, err)
 	}
 	return shrinkImage(output.Image{
-		Data:        data,
+		Data:        got.body,
 		ContentType: contentType,
 		Filename:    filename,
 	})
@@ -177,8 +222,8 @@ func (dist *distributor) created(event discord.Event) {
 	}

 	var preview *output.Preview
-	if videoURL := findYouTubeURL(event.Content); videoURL != "" {
-		preview, err = youtubePreview(videoURL)
+	if link := findLink(event.Content); link != "" {
+		preview, err = linkPreview(link)
 		if err != nil {
 			// The post is still worth making without its card.
 			fmt.Fprintln(os.Stderr, err)