package main
import (
"bytes"
"encoding/json"
"fmt"
"net/url"
"os"
"path"
"strings"
"tweetdistributor/output"
"golang.org/x/net/html"
"golang.org/x/net/html/charset"
)
// maxHTMLBytes caps how much of a page is read while looking for its preview
// tags. They belong in , so reading further is wasted.
const maxHTMLBytes = 1 << 20
// linkPreview builds the preview card for a link. Bluesky shows no card of
// its own: whatever the record does not embed is not displayed, so every card
// has to be assembled here.
func linkPreview(link string) (*output.Preview, error) {
u, err := url.Parse(link)
if err != nil {
return nil, fmt.Errorf("parsing %s: %w", link, err)
}
if isYouTube(u) {
return youtubePreview(u)
}
return ogpPreview(u)
}
// isYouTube reports whether u addresses a YouTube video.
func isYouTube(u *url.URL) bool {
switch strings.ToLower(u.Hostname()) {
case "youtu.be":
return strings.Trim(u.Path, "/") != ""
case "youtube.com", "www.youtube.com", "m.youtube.com", "music.youtube.com":
if u.Path == "/watch" && u.Query().Get("v") != "" {
return true
}
return strings.HasPrefix(u.Path, "/shorts/") || strings.HasPrefix(u.Path, "/live/")
}
return false
}
// oEmbedResponse is the part of YouTube's oEmbed document we care about.
type oEmbedResponse struct {
Title string `json:"title"`
AuthorName string `json:"author_name"`
ThumbnailURL string `json:"thumbnail_url"`
}
// youtubePreview builds the card for a YouTube link from the oEmbed endpoint,
// which answers with just the few fields a card needs instead of the megabyte
// of markup the watch page is.
func youtubePreview(video *url.URL) (*output.Preview, error) {
endpoint := "https://www.youtube.com/oembed?format=json&url=" + url.QueryEscape(video.String())
got, err := fetch(endpoint, maxHTMLBytes)
if err != nil {
return nil, fmt.Errorf("fetching preview for %s: %w", video, err)
}
var oembed oEmbedResponse
if err := json.Unmarshal(got.body, &oembed); err != nil {
return nil, fmt.Errorf("parsing preview for %s: %w", video, err)
}
preview := &output.Preview{
URL: video.String(),
Title: oembed.Title,
Description: oembed.AuthorName,
}
if oembed.ThumbnailURL != "" {
preview.Thumb = thumbnail(video, oembed.ThumbnailURL)
}
return preview, nil
}
// ogpPreview builds the card for an ordinary page from its Open Graph tags,
// falling back to the Twitter card tags and then to the plain document title.
func ogpPreview(page *url.URL) (*output.Preview, error) {
got, err := fetch(page.String(), maxHTMLBytes)
if err != nil {
return nil, fmt.Errorf("fetching preview for %s: %w", page, err)
}
switch got.mediaType {
case "", "text/html", "application/xhtml+xml":
default:
return nil, fmt.Errorf("%s is %s, which carries no preview tags", page, got.mediaType)
}
tags, err := parseMetaTags(got)
if err != nil {
return nil, fmt.Errorf("reading preview for %s: %w", page, err)
}
title := tags.first("og:title", "twitter:title", "title")
if title == "" {
return nil, fmt.Errorf("%s has no title to put on a card", page)
}
preview := &output.Preview{
// The card links to the page as it was written, not as it redirected.
URL: page.String(),
Title: title,
Description: tags.first("og:description", "twitter:description", "description"),
}
if image := tags.first("og:image", "og:image:url", "og:image:secure_url", "twitter:image", "twitter:image:src"); image != "" {
preview.Thumb = thumbnail(got.url, image)
}
return preview, nil
}
// metaTags holds a page's tags keyed by their property or name
// attribute, plus its under "title".
type metaTags map[string]string
// first returns the value of the earliest of keys that the page set.
func (tags metaTags) first(keys ...string) string {
for _, key := range keys {
if value := strings.TrimSpace(tags[key]); value != "" {
return value
}
}
return ""
}
// parseMetaTags reads the tags out of a page. It stops at <body>, past which
// preview tags do not belong, and treats running out of markup as the end:
// the body was cut off at maxHTMLBytes.
func parseMetaTags(page *fetched) (metaTags, error) {
// Pages are not all UTF-8; charset works out the encoding from the
// Content-Type header, a byte order mark or the meta charset tag.
decoded, err := charset.NewReader(bytes.NewReader(page.body), page.contentType)
if err != nil {
return nil, err
}
tags := metaTags{}
tokenizer := html.NewTokenizer(decoded)
for {
switch tokenizer.Next() {
case html.ErrorToken:
return tags, nil
case html.StartTagToken, html.SelfClosingTagToken:
name, hasattr := tokenizer.TagName()
switch string(name) {
case "meta":
if !hasattr {
continue
}
var key, content string
for {
attr, value, more := tokenizer.TagAttr()
switch string(attr) {
case "property", "name":
key = strings.ToLower(string(value))
case "content":
content = string(value)
}
if !more {
break
}
}
// The first tag of a name wins, as it does in every reader.
if key != "" && content != "" && tags[key] == "" {
tags[key] = content
}
case "title":
if tokenizer.Next() == html.TextToken && tags["title"] == "" {
tags["title"] = strings.TrimSpace(string(tokenizer.Text()))
}
case "body":
return tags, nil
}
}
}
}
// thumbnail fetches a card image, resolving ref against the page it was found
// on. A card without its picture is still worth posting, so a thumbnail that
// cannot be fetched is reported and dropped rather than failing the card.
func thumbnail(base *url.URL, ref string) *output.Image {
imageurl, err := base.Parse(ref)
if err != nil {
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", ref, err)
return nil
}
got, err := fetch(imageurl.String(), maxDownloadBytes)
if err != nil {
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
return nil
}
filename := path.Base(imageurl.Path)
if filename == "." || filename == "/" {
filename = "thumbnail"
}
img, err := shrinkImage(output.Image{
Data: got.body,
ContentType: got.mediaType,
Filename: filename,
})
if err != nil {
fmt.Fprintf(os.Stderr, "preview thumbnail %s: %s\n", imageurl, err)
return nil
}
return &img
}