diff --git a/go.mod b/go.mod
index 5a343cf..d150aa6 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
github.com/bluesky-social/indigo v0.0.0-20250313000755-d9a74f690c90
github.com/bwmarrin/discordgo v0.28.1
golang.org/x/image v0.43.0
+ golang.org/x/net v0.23.0
)
require (
@@ -51,6 +52,7 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
+ golang.org/x/text v0.38.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
)
diff --git a/go.sum b/go.sum
index 31080f2..4f6f7a0 100644
--- a/go.sum
+++ b/go.sum
@@ -178,6 +178,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -196,6 +198,8 @@ golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
+golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
diff --git a/main.go b/main.go
index 83cc5f0..c2a6310 100644
--- a/main.go
+++ b/main.go
@@ -8,7 +8,9 @@ import (
"image/jpeg"
_ "image/png"
"io"
+ "mime"
"net/http"
+ "net/url"
"os"
"path"
"regexp"
@@ -55,8 +57,31 @@ func tweetLength(content string) int {
return length
}
+// maxDownloadBytes caps an image download. Discord's own attachment limit is
+// well below it, so a truncated image means something else served us a body
+// far larger than any picture we would want to post.
+const maxDownloadBytes = 32 << 20
+
+// userAgent names the bot to the sites whose preview tags it reads; some of
+// them serve those tags only to a client that identifies itself.
+const userAgent = "tweetdistributor/1.0 (link preview)"
+
var httpClient = &http.Client{Timeout: 30 * time.Second}
+// fetched is a downloaded document together with what the response said
+// about it.
+type fetched struct {
+ body []byte
+ // mediaType is the Content-Type without its parameters, e.g. "text/html".
+ mediaType string
+ // contentType is the header as sent, parameters and all, which is what
+ // tells a decoder the character encoding.
+ contentType string
+ // url is where the body actually came from, after any redirects, and is
+ // what relative links in it resolve against.
+ url *url.URL
+}
+
// shrinkImage re-encodes (and if necessary downscales) an image until it
// fits within maxImageBytes. Images already small enough pass through
// untouched.
@@ -98,32 +123,52 @@ func shrinkImage(img output.Image) (output.Image, error) {
return output.Image{}, fmt.Errorf("%s could not be shrunk below %d bytes", img.Filename, maxImageBytes)
}
-// fetch GETs url and returns its body.
-func fetch(url string) ([]byte, error) {
- resp, err := httpClient.Get(url)
+// fetch GETs rawurl, reading at most limit bytes of the body. Callers that
+// only need the beginning of a document pass a small limit and treat the
+// truncation as normal.
+func fetch(rawurl string, limit int64) (*fetched, error) {
+ req, err := http.NewRequest(http.MethodGet, rawurl, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("User-Agent", userAgent)
+
+ resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
- data, err := io.ReadAll(resp.Body)
+ data, err := io.ReadAll(io.LimitReader(resp.Body, limit))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %s", resp.Status)
}
- return data, nil
+
+ contenttype := resp.Header.Get("Content-Type")
+ mediatype, _, err := mime.ParseMediaType(contenttype)
+ if err != nil {
+ mediatype = ""
+ }
+
+ return &fetched{
+ body: data,
+ mediaType: mediatype,
+ contentType: contenttype,
+ url: resp.Request.URL,
+ }, nil
}
// downloadImage fetches an image and shrinks it to a postable size.
func downloadImage(url, filename, contentType string) (output.Image, error) {
- data, err := fetch(url)
+ got, err := fetch(url, maxDownloadBytes)
if err != nil {
return output.Image{}, fmt.Errorf("downloading %s: %w", filename, err)
}
return shrinkImage(output.Image{
- Data: data,
+ Data: got.body,
ContentType: contentType,
Filename: filename,
})
@@ -177,8 +222,8 @@ func (dist *distributor) created(event discord.Event) {
}
var preview *output.Preview
- if videoURL := findYouTubeURL(event.Content); videoURL != "" {
- preview, err = youtubePreview(videoURL)
+ if link := findLink(event.Content); link != "" {
+ preview, err = linkPreview(link)
if err != nil {
// The post is still worth making without its card.
fmt.Fprintln(os.Stderr, err)
diff --git a/preview.go b/preview.go
index 713d7a6..718738a 100644
--- a/preview.go
+++ b/preview.go
@@ -1,39 +1,62 @@
package main
import (
+ "bytes"
"encoding/json"
"fmt"
"net/url"
+ "os"
+ "path"
"strings"
"tweetdistributor/output"
+
+ "golang.org/x/net/html"
+ "golang.org/x/net/html/charset"
)
-// findYouTubeURL returns the first YouTube video link in content, or "" if
-// there is none.
-func findYouTubeURL(content string) string {
+// maxHTMLBytes caps how much of a page is read while looking for its preview
+// tags. They belong in , so reading further is wasted.
+const maxHTMLBytes = 1 << 20
+
+// findLink returns the first link in content, or "" if there is none.
+func findLink(content string) string {
for _, match := range urlPattern.FindAllString(content, -1) {
raw := trimURL(match)
- u, err := url.Parse(raw)
- if err != nil {
- continue
- }
- switch strings.ToLower(u.Hostname()) {
- case "youtu.be":
- if strings.Trim(u.Path, "/") != "" {
- return raw
- }
- case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
- if u.Path == "/watch" && u.Query().Get("v") != "" {
- return raw
- }
- if strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/") {
- return raw
- }
+ if u, err := url.Parse(raw); err == nil && u.Host != "" {
+ return raw
}
}
return ""
}
+// linkPreview builds the preview card for a link. Bluesky shows no card of
+// its own: whatever the record does not embed is not displayed, so every card
+// has to be assembled here.
+func linkPreview(link string) (*output.Preview, error) {
+ u, err := url.Parse(link)
+ if err != nil {
+ return nil, fmt.Errorf("parsing %s: %w", link, err)
+ }
+ if isYouTube(u) {
+ return youtubePreview(u)
+ }
+ return ogpPreview(u)
+}
+
+// isYouTube reports whether u addresses a YouTube video.
+func isYouTube(u *url.URL) bool {
+ switch strings.ToLower(u.Hostname()) {
+ case "youtu.be":
+ return strings.Trim(u.Path, "/") != ""
+ case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
+ if u.Path == "/watch" && u.Query().Get("v") != "" {
+ return true
+ }
+ return strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/")
+ }
+ return false
+}
+
// oEmbedResponse is the part of YouTube's oEmbed document we care about.
type oEmbedResponse struct {
Title string `json:"title"`
@@ -41,34 +64,161 @@ type oEmbedResponse struct {
ThumbnailURL string `json:"thumbnail_url"`
}
-// youtubePreview builds the preview card for a YouTube link by asking
-// YouTube's oEmbed endpoint for the title, channel and thumbnail.
-func youtubePreview(videoURL string) (*output.Preview, error) {
- endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(videoURL)
- body, err := fetch(endpoint)
+// youtubePreview builds the card for a YouTube link from the oEmbed endpoint,
+// which answers with just the few fields a card needs instead of the megabyte
+// of markup the watch page is.
+func youtubePreview(video *url.URL) (*output.Preview, error) {
+ endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(video.String())
+ got, err := fetch(endpoint, maxHTMLBytes)
if err != nil {
- return nil, fmt.Errorf("fetching preview for %s: %w", videoURL, err)
+ return nil, fmt.Errorf("fetching preview for %s: %w", video, err)
}
var oembed oEmbedResponse
- if err := json.Unmarshal(body, &oembed); err != nil {
- return nil, fmt.Errorf("parsing preview for %s: %w", videoURL, err)
+ if err := json.Unmarshal(got.body, &oembed); err != nil {
+ return nil, fmt.Errorf("parsing preview for %s: %w", video, err)
}
preview := &output.Preview{
- URL: videoURL,
+ URL: video.String(),
Title: oembed.Title,
Description: oembed.AuthorName,
}
-
if oembed.ThumbnailURL != "" {
- thumb, err := downloadImage(oembed.ThumbnailURL, "thumbnail.jpg", "image/jpeg")
- if err != nil {
- // A card without its thumbnail is still worth posting.
- return preview, nil
- }
- preview.Thumb = &thumb
+ preview.Thumb = thumbnail(video, oembed.ThumbnailURL)
}
-
return preview, nil
}
+
+// ogpPreview builds the card for an ordinary page from its Open Graph tags,
+// falling back to the Twitter card tags and then to the plain document title.
+func ogpPreview(page *url.URL) (*output.Preview, error) {
+ got, err := fetch(page.String(), maxHTMLBytes)
+ if err != nil {
+ return nil, fmt.Errorf("fetching preview for %s: %w", page, err)
+ }
+ switch got.mediaType {
+ case "", "text/html", "application/xhtml+xml":
+ default:
+ return nil, fmt.Errorf("%s is %s, which carries no preview tags", page, got.mediaType)
+ }
+
+ tags, err := parseMetaTags(got)
+ if err != nil {
+ return nil, fmt.Errorf("reading preview for %s: %w", page, err)
+ }
+
+ title := tags.first("og:title", "twitter:title", "title")
+ if title == "" {
+ return nil, fmt.Errorf("%s has no title to put on a card", page)
+ }
+
+ preview := &output.Preview{
+ // The card links to the page as it was written, not as it redirected.
+ URL: page.String(),
+ Title: title,
+ Description: tags.first("og:description", "twitter:description", "description"),
+ }
+ if image := tags.first("og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src"); image != "" {
+ preview.Thumb = thumbnail(got.url, image)
+ }
+ return preview, nil
+}
+
+// metaTags holds a page's tags keyed by their property or name
+// attribute, plus its under "title".
+type metaTags map[string]string
+
+// first returns the value of the earliest of keys that the page set.
+func (tags metaTags) first(keys ...string) string {
+ for _, key := range keys {
+ if value := strings.TrimSpace(tags[key]); value != "" {
+ return value
+ }
+ }
+ return ""
+}
+
+// parseMetaTags reads the tags out of a page. It stops at <body>, past which
+// preview tags do not belong, and treats running out of markup as the end:
+// the body was cut off at maxHTMLBytes.
+func parseMetaTags(page *fetched) (metaTags, error) {
+ // Pages are not all UTF-8; charset works out the encoding from the
+ // Content-Type header, a byte order mark or the meta charset tag.
+ decoded, err := charset.NewReader(bytes.NewReader(page.body), page.contentType)
+ if err != nil {
+ return nil, err
+ }
+
+ tags := metaTags{}
+ tokenizer := html.NewTokenizer(decoded)
+ for {
+ switch tokenizer.Next() {
+ case html.ErrorToken:
+ return tags, nil
+ case html.StartTagToken, html.SelfClosingTagToken:
+ name, hasattr := tokenizer.TagName()
+ switch string(name) {
+ case "meta":
+ if !hasattr {
+ continue
+ }
+ var key, content string
+ for {
+ attr, value, more := tokenizer.TagAttr()
+ switch string(attr) {
+ case "property", "name":
+ key = strings.ToLower(string(value))
+ case "content":
+ content = string(value)
+ }
+ if !more {
+ break
+ }
+ }
+ // The first tag of a name wins, as it does in every reader.
+ if key != "" && content != "" && tags[key] == "" {
+ tags[key] = content
+ }
+ case "title":
+ if tokenizer.Next() == html.TextToken && tags["title"] == "" {
+ tags["title"] = strings.TrimSpace(string(tokenizer.Text()))
+ }
+ case "body":
+ return tags, nil
+ }
+ }
+ }
+}
+
+// thumbnail fetches a card image, resolving ref against the page it was found
+// on. A card without its picture is still worth posting, so a thumbnail that
+// cannot be fetched is reported and dropped rather than failing the card.
+func thumbnail(base *url.URL, ref string) *output.Image {
+ imageurl, err := base.Parse(ref)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", ref, err)
+ return nil
+ }
+
+ got, err := fetch(imageurl.String(), maxDownloadBytes)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
+ return nil
+ }
+
+ filename := path.Base(imageurl.Path)
+ if filename == "." || filename == "/" {
+ filename = "thumbnail"
+ }
+ img, err := shrinkImage(output.Image{
+ Data: got.body,
+ ContentType: got.mediaType,
+ Filename: filename,
+ })
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
+ return nil
+ }
+ return &img
+}
diff --git a/preview_test.go b/preview_test.go
index 852b9bb..fd08ec3 100644
--- a/preview_test.go
+++ b/preview_test.go
@@ -1,33 +1,133 @@
package main
-import "testing"
+import (
+ "net/url"
+ "testing"
+)
-func TestFindYouTubeURL(t *testing.T) {
+func TestFindLink(t *testing.T) {
tests := []struct {
name string
content string
want string
}{
- {"watch", "みてみて https://www.youtube.com/watch?v=dQw4w9WgXcQ おもしろい", "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
- {"short host", "https://youtu.be/dQw4w9WgXcQ?t=42", "https://youtu.be/dQw4w9WgXcQ?t=42"},
- {"shorts", "https://www.youtube.com/shorts/abc_123", "https://www.youtube.com/shorts/abc_123"},
- {"live", "https://youtube.com/live/abc-123", "https://youtube.com/live/abc-123"},
- {"mobile", "https://m.youtube.com/watch?v=abc&feature=share", "https://m.youtube.com/watch?v=abc&feature=share"},
- {"trailing punctuation", "これ→https://youtu.be/abc123。", "https://youtu.be/abc123"},
- {"first of several", "https://youtu.be/one https://youtu.be/two", "https://youtu.be/one"},
- {"skips other links", "https://example.com/watch?v=x https://youtu.be/abc", "https://youtu.be/abc"},
- {"no url", "ただのつぶやき", ""},
- {"other site", "https://example.com/", ""},
- {"youtube without video", "https://www.youtube.com/", ""},
- {"watch without v", "https://www.youtube.com/watch?list=PL123", ""},
- {"lookalike host", "https://youtube.com.evil.example/watch?v=abc", ""},
+ {"plain", "みてみて https://example.com/article おもしろい", "https://example.com/article"},
+ {"query", "https://example.com/watch?v=abc&t=1", "https://example.com/watch?v=abc&t=1"},
+ {"trailing punctuation", "これ→https://example.com/記事。", "https://example.com/記事"},
+ {"first of several", "https://one.example https://two.example", "https://one.example"},
+ {"no link", "ただのつぶやき", ""},
+ {"scheme only", "http:// と書いただけ", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- if got := findYouTubeURL(tt.content); got != tt.want {
- t.Errorf("findYouTubeURL(%q) = %q, want %q", tt.content, got, tt.want)
+ if got := findLink(tt.content); got != tt.want {
+ t.Errorf("findLink(%q) = %q, want %q", tt.content, got, tt.want)
}
})
}
}
+
+func TestIsYouTube(t *testing.T) {
+ tests := []struct {
+ raw string
+ want bool
+ }{
+ {"https://www.youtube.com/watch?v=dQw4w9WgXcQ", true},
+ {"https://youtu.be/dQw4w9WgXcQ?t=42", true},
+ {"https://www.youtube.com/shorts/abc_123", true},
+ {"https://youtube.com/live/abc-123", true},
+ {"https://m.youtube.com/watch?v=abc&feature=share", true},
+ {"https://www.youtube.com/", false},
+ {"https://www.youtube.com/watch?list=PL123", false},
+ {"https://youtube.com.evil.example/watch?v=abc", false},
+ {"https://example.com/", false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.raw, func(t *testing.T) {
+ u, err := url.Parse(tt.raw)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := isYouTube(u); got != tt.want {
+ t.Errorf("isYouTube(%q) = %v, want %v", tt.raw, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestParseMetaTags(t *testing.T) {
+ page := &fetched{
+ contentType: "text/html; charset=utf-8",
+ body: []byte(`<!doctype html><html><head>
+ <title>plain title
+
+
+
+
+
+
+
+