sirrow
953664ff1c
Bluesky renders no link card of its own. Its PDS never fetches the page behind a URL, so a post whose record has no app.bsky.embed.external shows the link as bare text; every client that shows a card builds it before posting. We only built one for YouTube, so every other link arrived on bluesky with nothing attached. preview.go now reads the Open Graph tags of an ordinary page and turns them into the same card: og:title and og:description, falling back to the twitter:* tags and then to <title> and meta description, with og:image fetched as the thumbnail. YouTube keeps its oEmbed path, which answers with the handful of fields a card needs rather than the megabyte of markup the watch page is. Only the head of a page is read, and parsing stops at <body>, since preview tags belong above it and a truncated page still yields what was read. Pages are decoded through x/net/html/charset rather than assumed to be UTF-8, which Japanese pages served as Shift_JIS are not. The thumbnail is resolved against the URL the body came from, so a relative og:image survives a redirect. Requests now name the bot in a User-Agent, which some sites want before serving preview tags at all. fetch grew a byte limit and now reports the media type and final URL its body came with, which is what the thumbnail needs to name and resolve itself. Checked against go.dev, Japanese Wikipedia and a YouTube video. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
225 lines
6.5 KiB
Go
225 lines
6.5 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"tweetdistributor/output"
|
|
|
|
"golang.org/x/net/html"
|
|
"golang.org/x/net/html/charset"
|
|
)
|
|
|
|
// maxHTMLBytes caps how much of a page is read while looking for its preview
|
|
// tags. They belong in <head>, so reading further is wasted.
|
|
const maxHTMLBytes = 1 << 20
|
|
|
|
// findLink returns the first link in content, or "" if there is none.
|
|
func findLink(content string) string {
|
|
for _, match := range urlPattern.FindAllString(content, -1) {
|
|
raw := trimURL(match)
|
|
if u, err := url.Parse(raw); err == nil && u.Host != "" {
|
|
return raw
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// linkPreview builds the preview card for a link. Bluesky shows no card of
|
|
// its own: whatever the record does not embed is not displayed, so every card
|
|
// has to be assembled here.
|
|
func linkPreview(link string) (*output.Preview, error) {
|
|
u, err := url.Parse(link)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing %s: %w", link, err)
|
|
}
|
|
if isYouTube(u) {
|
|
return youtubePreview(u)
|
|
}
|
|
return ogpPreview(u)
|
|
}
|
|
|
|
// isYouTube reports whether u addresses a YouTube video.
|
|
func isYouTube(u *url.URL) bool {
|
|
switch strings.ToLower(u.Hostname()) {
|
|
case "youtu.be":
|
|
return strings.Trim(u.Path, "/") != ""
|
|
case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
|
|
if u.Path == "/watch" && u.Query().Get("v") != "" {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/")
|
|
}
|
|
return false
|
|
}
|
|
|
|
// oEmbedResponse is the part of YouTube's oEmbed document we care about.
|
|
type oEmbedResponse struct {
|
|
Title string `json:"title"`
|
|
AuthorName string `json:"author_name"`
|
|
ThumbnailURL string `json:"thumbnail_url"`
|
|
}
|
|
|
|
// youtubePreview builds the card for a YouTube link from the oEmbed endpoint,
|
|
// which answers with just the few fields a card needs instead of the megabyte
|
|
// of markup the watch page is.
|
|
func youtubePreview(video *url.URL) (*output.Preview, error) {
|
|
endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(video.String())
|
|
got, err := fetch(endpoint, maxHTMLBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching preview for %s: %w", video, err)
|
|
}
|
|
|
|
var oembed oEmbedResponse
|
|
if err := json.Unmarshal(got.body, &oembed); err != nil {
|
|
return nil, fmt.Errorf("parsing preview for %s: %w", video, err)
|
|
}
|
|
|
|
preview := &output.Preview{
|
|
URL: video.String(),
|
|
Title: oembed.Title,
|
|
Description: oembed.AuthorName,
|
|
}
|
|
if oembed.ThumbnailURL != "" {
|
|
preview.Thumb = thumbnail(video, oembed.ThumbnailURL)
|
|
}
|
|
return preview, nil
|
|
}
|
|
|
|
// ogpPreview builds the card for an ordinary page from its Open Graph tags,
|
|
// falling back to the Twitter card tags and then to the plain document title.
|
|
func ogpPreview(page *url.URL) (*output.Preview, error) {
|
|
got, err := fetch(page.String(), maxHTMLBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching preview for %s: %w", page, err)
|
|
}
|
|
switch got.mediaType {
|
|
case "", "text/html", "application/xhtml+xml":
|
|
default:
|
|
return nil, fmt.Errorf("%s is %s, which carries no preview tags", page, got.mediaType)
|
|
}
|
|
|
|
tags, err := parseMetaTags(got)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading preview for %s: %w", page, err)
|
|
}
|
|
|
|
title := tags.first("og:title", "twitter:title", "title")
|
|
if title == "" {
|
|
return nil, fmt.Errorf("%s has no title to put on a card", page)
|
|
}
|
|
|
|
preview := &output.Preview{
|
|
// The card links to the page as it was written, not as it redirected.
|
|
URL: page.String(),
|
|
Title: title,
|
|
Description: tags.first("og:description", "twitter:description", "description"),
|
|
}
|
|
if image := tags.first("og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src"); image != "" {
|
|
preview.Thumb = thumbnail(got.url, image)
|
|
}
|
|
return preview, nil
|
|
}
|
|
|
|
// metaTags holds a page's <meta> tags keyed by their property or name
|
|
// attribute, plus its <title> under "title".
|
|
type metaTags map[string]string
|
|
|
|
// first returns the value of the earliest of keys that the page set.
|
|
func (tags metaTags) first(keys ...string) string {
|
|
for _, key := range keys {
|
|
if value := strings.TrimSpace(tags[key]); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// parseMetaTags reads the tags out of a page. It stops at <body>, past which
|
|
// preview tags do not belong, and treats running out of markup as the end:
|
|
// the body was cut off at maxHTMLBytes.
|
|
func parseMetaTags(page *fetched) (metaTags, error) {
|
|
// Pages are not all UTF-8; charset works out the encoding from the
|
|
// Content-Type header, a byte order mark or the meta charset tag.
|
|
decoded, err := charset.NewReader(bytes.NewReader(page.body), page.contentType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tags := metaTags{}
|
|
tokenizer := html.NewTokenizer(decoded)
|
|
for {
|
|
switch tokenizer.Next() {
|
|
case html.ErrorToken:
|
|
return tags, nil
|
|
case html.StartTagToken, html.SelfClosingTagToken:
|
|
name, hasattr := tokenizer.TagName()
|
|
switch string(name) {
|
|
case "meta":
|
|
if !hasattr {
|
|
continue
|
|
}
|
|
var key, content string
|
|
for {
|
|
attr, value, more := tokenizer.TagAttr()
|
|
switch string(attr) {
|
|
case "property", "name":
|
|
key = strings.ToLower(string(value))
|
|
case "content":
|
|
content = string(value)
|
|
}
|
|
if !more {
|
|
break
|
|
}
|
|
}
|
|
// The first tag of a name wins, as it does in every reader.
|
|
if key != "" && content != "" && tags[key] == "" {
|
|
tags[key] = content
|
|
}
|
|
case "title":
|
|
if tokenizer.Next() == html.TextToken && tags["title"] == "" {
|
|
tags["title"] = strings.TrimSpace(string(tokenizer.Text()))
|
|
}
|
|
case "body":
|
|
return tags, nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// thumbnail fetches a card image, resolving ref against the page it was found
|
|
// on. A card without its picture is still worth posting, so a thumbnail that
|
|
// cannot be fetched is reported and dropped rather than failing the card.
|
|
func thumbnail(base *url.URL, ref string) *output.Image {
|
|
imageurl, err := base.Parse(ref)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", ref, err)
|
|
return nil
|
|
}
|
|
|
|
got, err := fetch(imageurl.String(), maxDownloadBytes)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
|
|
return nil
|
|
}
|
|
|
|
filename := path.Base(imageurl.Path)
|
|
if filename == "." || filename == "/" {
|
|
filename = "thumbnail"
|
|
}
|
|
img, err := shrinkImage(output.Image{
|
|
Data: got.body,
|
|
ContentType: got.mediaType,
|
|
Filename: filename,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
|
|
return nil
|
|
}
|
|
return &img
|
|
}
|