0.29.1原版

2026-07-02 19:14:14 +08:00
commit d94008f0fb
947 changed files with 174905 additions and 0 deletions
@@ -0,0 +1,227 @@
+package httpgetter
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+var ErrInternalIP = errors.New("internal IP addresses are not allowed")
+
+const maxHTMLMetaBytes = 512 * 1024
+
+var (
+	lookupIPAddr = net.DefaultResolver.LookupIPAddr
+	dialContext  = (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}).DialContext
+	httpClient = newHTTPClient()
+)
+
+func newHTTPClient() *http.Client {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.Proxy = nil
+	transport.DialContext = secureDialContext
+
+	return &http.Client{
+		Transport: transport,
+		Timeout:   5 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if err := validateURL(req.URL.String()); err != nil {
+				return errors.Wrap(err, "redirect to internal IP")
+			}
+			if len(via) >= 10 {
+				return errors.New("too many redirects")
+			}
+			return nil
+		},
+	}
+}
+
+func secureDialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid address")
+	}
+
+	ips, err := resolveAllowedIPs(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+
+	var dialErr error
+	for _, ip := range ips {
+		conn, err := dialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+		if err == nil {
+			return conn, nil
+		}
+		dialErr = err
+	}
+	return nil, dialErr
+}
+
+func resolveAllowedIPs(ctx context.Context, host string) ([]net.IP, error) {
+	if ip := net.ParseIP(host); ip != nil {
+		if isInternalIP(ip) {
+			return nil, errors.Wrap(ErrInternalIP, ip.String())
+		}
+		return []net.IP{ip}, nil
+	}
+
+	addrs, err := lookupIPAddr(ctx, host)
+	if err != nil {
+		return nil, errors.Errorf("failed to resolve hostname: %v", err)
+	}
+
+	ips := make([]net.IP, 0, len(addrs))
+	for _, addr := range addrs {
+		ip := addr.IP
+		if ip == nil {
+			continue
+		}
+		if isInternalIP(ip) {
+			return nil, errors.Wrapf(ErrInternalIP, "host=%s, ip=%s", host, ip.String())
+		}
+		ips = append(ips, ip)
+	}
+	if len(ips) == 0 {
+		return nil, errors.New("hostname resolved to no addresses")
+	}
+
+	return ips, nil
+}
+
+func isInternalIP(ip net.IP) bool {
+	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()
+}
+
+func validateURL(urlStr string) error {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return errors.New("invalid URL format")
+	}
+
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return errors.New("only http/https protocols are allowed")
+	}
+
+	host := u.Hostname()
+	if host == "" {
+		return errors.New("empty hostname")
+	}
+
+	if ip := net.ParseIP(host); ip != nil && isInternalIP(ip) {
+		return errors.Wrap(ErrInternalIP, ip.String())
+	}
+
+	return nil
+}
+
+type HTMLMeta struct {
+	Title       string `json:"title"`
+	Description string `json:"description"`
+	Image       string `json:"image"`
+}
+
+func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
+	if err := validateURL(urlStr); err != nil {
+		return nil, err
+	}
+
+	response, err := httpClient.Get(urlStr)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	mediatype, err := getMediatype(response)
+	if err != nil {
+		return nil, err
+	}
+	if mediatype != "text/html" {
+		return nil, errors.New("not a HTML page")
+	}
+
+	htmlMeta := extractHTMLMeta(io.LimitReader(response.Body, maxHTMLMetaBytes))
+	enrichSiteMeta(response.Request.URL, htmlMeta)
+	return htmlMeta, nil
+}
+
+func extractHTMLMeta(resp io.Reader) *HTMLMeta {
+	tokenizer := html.NewTokenizer(resp)
+	htmlMeta := new(HTMLMeta)
+
+	for {
+		tokenType := tokenizer.Next()
+		if tokenType == html.ErrorToken {
+			break
+		} else if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken {
+			token := tokenizer.Token()
+			if token.DataAtom == atom.Body {
+				break
+			}
+
+			if token.DataAtom == atom.Title {
+				tokenizer.Next()
+				token := tokenizer.Token()
+				htmlMeta.Title = token.Data
+			} else if token.DataAtom == atom.Meta {
+				ogTitle, ok := extractMetaProperty(token, "og:title")
+				if ok {
+					htmlMeta.Title = ogTitle
+				}
+
+				ogDescription, ok := extractMetaProperty(token, "og:description")
+				if ok {
+					htmlMeta.Description = ogDescription
+				}
+
+				ogImage, ok := extractMetaProperty(token, "og:image")
+				if ok {
+					htmlMeta.Image = ogImage
+				}
+
+				description, ok := extractMetaProperty(token, "description")
+				if ok && htmlMeta.Description == "" {
+					htmlMeta.Description = description
+				}
+			}
+		}
+	}
+
+	return htmlMeta
+}
+
+func extractMetaProperty(token html.Token, prop string) (content string, ok bool) {
+	content, ok = "", false
+	for _, attr := range token.Attr {
+		if (attr.Key == "property" || attr.Key == "name") && strings.EqualFold(attr.Val, prop) {
+			ok = true
+		}
+		if attr.Key == "content" {
+			content = attr.Val
+		}
+	}
+	return content, ok
+}
+
+func enrichSiteMeta(url *url.URL, meta *HTMLMeta) {
+	if url.Hostname() == "www.youtube.com" {
+		if url.Path == "/watch" {
+			vid := url.Query().Get("v")
+			if vid != "" {
+				meta.Image = fmt.Sprintf("https://img.youtube.com/vi/%s/mqdefault.jpg", vid)
+			}
+		}
+	}
+}
@@ -0,0 +1,189 @@
+package httpgetter
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func TestGetHTMLMeta(t *testing.T) {
+	originalHTTPClient := httpClient
+	t.Cleanup(func() {
+		httpClient = originalHTTPClient
+	})
+
+	httpClient = &http.Client{
+		Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			require.Equal(t, "http://93.184.216.34/article", req.URL.String())
+			return &http.Response{
+				StatusCode: http.StatusOK,
+				Header:     http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
+				Body: io.NopCloser(strings.NewReader(`<!doctype html>
+<html>
+<head>
+  <title>Fallback title</title>
+  <meta name="description" content="Fallback description">
+  <meta property="og:title" content="Open Graph title">
+  <meta property="og:description" content="Open Graph description">
+  <meta property="og:image" content="https://example.com/cover.png">
+</head>
+<body>ignored</body>
+</html>`)),
+				Request: req,
+			}, nil
+		}),
+	}
+
+	metadata, err := GetHTMLMeta("http://93.184.216.34/article")
+	require.NoError(t, err)
+	require.Equal(t, HTMLMeta{
+		Title:       "Open Graph title",
+		Description: "Open Graph description",
+		Image:       "https://example.com/cover.png",
+	}, *metadata)
+}
+
+func TestGetHTMLMetaWithNameOnly(t *testing.T) {
+	originalHTTPClient := httpClient
+	t.Cleanup(func() {
+		httpClient = originalHTTPClient
+	})
+
+	httpClient = &http.Client{
+		Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
+			return &http.Response{
+				StatusCode: http.StatusOK,
+				Header:     http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
+				Body: io.NopCloser(strings.NewReader(`<!doctype html>
+<html>
+<head>
+  <title>Sample Page</title>
+  <meta name="description" content="This description should appear in the link preview.">
+</head>
+<body>Hello</body>
+</html>`)),
+				Request: req,
+			}, nil
+		}),
+	}
+
+	metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
+	require.NoError(t, err)
+	require.Equal(t, HTMLMeta{
+		Title:       "Sample Page",
+		Description: "This description should appear in the link preview.",
+		Image:       "",
+	}, *metadata)
+}
+
+func TestGetHTMLMetaWithNameCaseInsensitive(t *testing.T) {
+	originalHTTPClient := httpClient
+	t.Cleanup(func() {
+		httpClient = originalHTTPClient
+	})
+
+	httpClient = &http.Client{
+		Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			require.Equal(t, "http://93.184.216.34/blog", req.URL.String())
+			return &http.Response{
+				StatusCode: http.StatusOK,
+				Header:     http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
+				Body: io.NopCloser(strings.NewReader(`<!doctype html>
+<html>
+<head>
+  <title>Sample Page</title>
+  <meta name="Description" content="Case insensitive description match.">
+</head>
+<body>Hello</body>
+</html>`)),
+				Request: req,
+			}, nil
+		}),
+	}
+
+	metadata, err := GetHTMLMeta("http://93.184.216.34/blog")
+	require.NoError(t, err)
+	require.Equal(t, HTMLMeta{
+		Title:       "Sample Page",
+		Description: "Case insensitive description match.",
+		Image:       "",
+	}, *metadata)
+}
+
+func TestGetHTMLMetaForInternal(t *testing.T) {
+	// test for internal IP
+	if _, err := GetHTMLMeta("http://192.168.0.1"); !errors.Is(err, ErrInternalIP) {
+		t.Errorf("Expected error for internal IP, got %v", err)
+	}
+
+	// test for resolved internal IP
+	if _, err := GetHTMLMeta("http://localhost"); !errors.Is(err, ErrInternalIP) {
+		t.Errorf("Expected error for resolved internal IP, got %v", err)
+	}
+}
+
+func TestHTTPClientHasTimeout(t *testing.T) {
+	require.NotZero(t, httpClient.Timeout)
+}
+
+func TestSecureDialContextRejectsResolvedInternalIP(t *testing.T) {
+	originalLookupIPAddr := lookupIPAddr
+	originalDialContext := dialContext
+	t.Cleanup(func() {
+		lookupIPAddr = originalLookupIPAddr
+		dialContext = originalDialContext
+	})
+
+	lookupIPAddr = func(context.Context, string) ([]net.IPAddr, error) {
+		return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}}, nil
+	}
+	dialContext = func(context.Context, string, string) (net.Conn, error) {
+		t.Fatal("internal IP should be rejected before dialing")
+		return nil, nil
+	}
+
+	_, err := secureDialContext(context.Background(), "tcp", "rebind.example:80")
+	require.ErrorIs(t, err, ErrInternalIP)
+}
+
+func TestSecureDialContextDialsResolvedIP(t *testing.T) {
+	originalLookupIPAddr := lookupIPAddr
+	originalDialContext := dialContext
+	t.Cleanup(func() {
+		lookupIPAddr = originalLookupIPAddr
+		dialContext = originalDialContext
+	})
+
+	lookupIPAddr = func(context.Context, string) ([]net.IPAddr, error) {
+		return []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, nil
+	}
+
+	var dialedAddress string
+	dialContext = func(_ context.Context, _ string, address string) (net.Conn, error) {
+		dialedAddress = address
+		clientConn, serverConn := net.Pipe()
+		t.Cleanup(func() {
+			clientConn.Close()
+			serverConn.Close()
+		})
+		return clientConn, nil
+	}
+
+	conn, err := secureDialContext(context.Background(), "tcp", "rebind.example:80")
+	require.NoError(t, err)
+	require.NotNil(t, conn)
+	require.Equal(t, "93.184.216.34:80", dialedAddress)
+}
@@ -0,0 +1 @@
+package httpgetter
@@ -0,0 +1,45 @@
+package httpgetter
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+type Image struct {
+	Blob      []byte
+	Mediatype string
+}
+
+func GetImage(urlStr string) (*Image, error) {
+	if _, err := url.Parse(urlStr); err != nil {
+		return nil, err
+	}
+
+	response, err := http.Get(urlStr)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	mediatype, err := getMediatype(response)
+	if err != nil {
+		return nil, err
+	}
+	if !strings.HasPrefix(mediatype, "image/") {
+		return nil, errors.New("wrong image mediatype")
+	}
+
+	bodyBytes, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	image := &Image{
+		Blob:      bodyBytes,
+		Mediatype: mediatype,
+	}
+	return image, nil
+}
@@ -0,0 +1,15 @@
+package httpgetter
+
+import (
+	"mime"
+	"net/http"
+)
+
+func getMediatype(response *http.Response) (string, error) {
+	contentType := response.Header.Get("content-type")
+	mediatype, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return "", err
+	}
+	return mediatype, nil
+}