sirrow
d794f65673
Bluesky linkifies nothing by itself. A URL in the text of a record is plain text unless a richtext facet says which bytes of the post are a link and where they point, so our posts carried URLs no one could press. Each link now travels on the Post as an output.Link with the offsets of the text it occupies, and the bluesky output turns those into app.bsky.richtext.facet#link. The offsets are UTF-8 byte offsets, not character counts: a facet measured in characters slides off the URL as soon as any Japanese text precedes it, and underlines the wrong words. Scanning moved from preview.go to findLinks in main.go, which now reports every link with its offsets rather than just the first one; the preview card still goes to the first, which is the one a reader meets first. X is unaffected, as it linkifies URLs itself. The facet feature only gains its lexicon type when marshalled, so the test checks the encoded record rather than the struct. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
214 lines
6.2 KiB
Go
214 lines
6.2 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"tweetdistributor/output"
|
|
|
|
"golang.org/x/net/html"
|
|
"golang.org/x/net/html/charset"
|
|
)
|
|
|
|
// maxHTMLBytes caps how much of a page is read while looking for its preview
|
|
// tags. They belong in <head>, so reading further is wasted.
|
|
const maxHTMLBytes = 1 << 20
|
|
|
|
// linkPreview builds the preview card for a link. Bluesky shows no card of
|
|
// its own: whatever the record does not embed is not displayed, so every card
|
|
// has to be assembled here.
|
|
func linkPreview(link string) (*output.Preview, error) {
|
|
u, err := url.Parse(link)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parsing %s: %w", link, err)
|
|
}
|
|
if isYouTube(u) {
|
|
return youtubePreview(u)
|
|
}
|
|
return ogpPreview(u)
|
|
}
|
|
|
|
// isYouTube reports whether u addresses a YouTube video.
|
|
func isYouTube(u *url.URL) bool {
|
|
switch strings.ToLower(u.Hostname()) {
|
|
case "youtu.be":
|
|
return strings.Trim(u.Path, "/") != ""
|
|
case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
|
|
if u.Path == "/watch" && u.Query().Get("v") != "" {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/")
|
|
}
|
|
return false
|
|
}
|
|
|
|
// oEmbedResponse is the part of YouTube's oEmbed document we care about.
|
|
type oEmbedResponse struct {
|
|
Title string `json:"title"`
|
|
AuthorName string `json:"author_name"`
|
|
ThumbnailURL string `json:"thumbnail_url"`
|
|
}
|
|
|
|
// youtubePreview builds the card for a YouTube link from the oEmbed endpoint,
|
|
// which answers with just the few fields a card needs instead of the megabyte
|
|
// of markup the watch page is.
|
|
func youtubePreview(video *url.URL) (*output.Preview, error) {
|
|
endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(video.String())
|
|
got, err := fetch(endpoint, maxHTMLBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching preview for %s: %w", video, err)
|
|
}
|
|
|
|
var oembed oEmbedResponse
|
|
if err := json.Unmarshal(got.body, &oembed); err != nil {
|
|
return nil, fmt.Errorf("parsing preview for %s: %w", video, err)
|
|
}
|
|
|
|
preview := &output.Preview{
|
|
URL: video.String(),
|
|
Title: oembed.Title,
|
|
Description: oembed.AuthorName,
|
|
}
|
|
if oembed.ThumbnailURL != "" {
|
|
preview.Thumb = thumbnail(video, oembed.ThumbnailURL)
|
|
}
|
|
return preview, nil
|
|
}
|
|
|
|
// ogpPreview builds the card for an ordinary page from its Open Graph tags,
|
|
// falling back to the Twitter card tags and then to the plain document title.
|
|
func ogpPreview(page *url.URL) (*output.Preview, error) {
|
|
got, err := fetch(page.String(), maxHTMLBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("fetching preview for %s: %w", page, err)
|
|
}
|
|
switch got.mediaType {
|
|
case "", "text/html", "application/xhtml+xml":
|
|
default:
|
|
return nil, fmt.Errorf("%s is %s, which carries no preview tags", page, got.mediaType)
|
|
}
|
|
|
|
tags, err := parseMetaTags(got)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading preview for %s: %w", page, err)
|
|
}
|
|
|
|
title := tags.first("og:title", "twitter:title", "title")
|
|
if title == "" {
|
|
return nil, fmt.Errorf("%s has no title to put on a card", page)
|
|
}
|
|
|
|
preview := &output.Preview{
|
|
// The card links to the page as it was written, not as it redirected.
|
|
URL: page.String(),
|
|
Title: title,
|
|
Description: tags.first("og:description", "twitter:description", "description"),
|
|
}
|
|
if image := tags.first("og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src"); image != "" {
|
|
preview.Thumb = thumbnail(got.url, image)
|
|
}
|
|
return preview, nil
|
|
}
|
|
|
|
// metaTags holds a page's <meta> tags keyed by their property or name
|
|
// attribute, plus its <title> under "title".
|
|
type metaTags map[string]string
|
|
|
|
// first returns the value of the earliest of keys that the page set.
|
|
func (tags metaTags) first(keys ...string) string {
|
|
for _, key := range keys {
|
|
if value := strings.TrimSpace(tags[key]); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// parseMetaTags reads the tags out of a page. It stops at <body>, past which
|
|
// preview tags do not belong, and treats running out of markup as the end:
|
|
// the body was cut off at maxHTMLBytes.
|
|
func parseMetaTags(page *fetched) (metaTags, error) {
|
|
// Pages are not all UTF-8; charset works out the encoding from the
|
|
// Content-Type header, a byte order mark or the meta charset tag.
|
|
decoded, err := charset.NewReader(bytes.NewReader(page.body), page.contentType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tags := metaTags{}
|
|
tokenizer := html.NewTokenizer(decoded)
|
|
for {
|
|
switch tokenizer.Next() {
|
|
case html.ErrorToken:
|
|
return tags, nil
|
|
case html.StartTagToken, html.SelfClosingTagToken:
|
|
name, hasattr := tokenizer.TagName()
|
|
switch string(name) {
|
|
case "meta":
|
|
if !hasattr {
|
|
continue
|
|
}
|
|
var key, content string
|
|
for {
|
|
attr, value, more := tokenizer.TagAttr()
|
|
switch string(attr) {
|
|
case "property", "name":
|
|
key = strings.ToLower(string(value))
|
|
case "content":
|
|
content = string(value)
|
|
}
|
|
if !more {
|
|
break
|
|
}
|
|
}
|
|
// The first tag of a name wins, as it does in every reader.
|
|
if key != "" && content != "" && tags[key] == "" {
|
|
tags[key] = content
|
|
}
|
|
case "title":
|
|
if tokenizer.Next() == html.TextToken && tags["title"] == "" {
|
|
tags["title"] = strings.TrimSpace(string(tokenizer.Text()))
|
|
}
|
|
case "body":
|
|
return tags, nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// thumbnail fetches a card image, resolving ref against the page it was found
|
|
// on. A card without its picture is still worth posting, so a thumbnail that
|
|
// cannot be fetched is reported and dropped rather than failing the card.
|
|
func thumbnail(base *url.URL, ref string) *output.Image {
|
|
imageurl, err := base.Parse(ref)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", ref, err)
|
|
return nil
|
|
}
|
|
|
|
got, err := fetch(imageurl.String(), maxDownloadBytes)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
|
|
return nil
|
|
}
|
|
|
|
filename := path.Base(imageurl.Path)
|
|
if filename == "." || filename == "/" {
|
|
filename = "thumbnail"
|
|
}
|
|
img, err := shrinkImage(output.Image{
|
|
Data: got.body,
|
|
ContentType: got.mediaType,
|
|
Filename: filename,
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
|
|
return nil
|
|
}
|
|
return &img
|
|
}
|