skeleton of embedr service, based on bskyweb

bskyweb/cmd/embedr/.gitignore

@@ -0,0 +1 @@
+/bskyweb

bskyweb/cmd/embedr/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"os"
+
+	_ "github.com/joho/godotenv/autoload"
+
+	logging "github.com/ipfs/go-log"
+	"github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("embedr")
+
+func init() {
+	logging.SetAllLoggers(logging.LevelDebug)
+	//logging.SetAllLoggers(logging.LevelWarn)
+}
+
+func main() {
+	run(os.Args)
+}
+
+func run(args []string) {
+
+	app := cli.App{
+		Name:  "embedr",
+		Usage: "web server for embed.bsky.app post embeds",
+	}
+
+	app.Commands = []*cli.Command{
+		&cli.Command{
+			Name:   "serve",
+			Usage:  "run the server",
+			Action: serve,
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:    "appview-host",
+					Usage:   "method, hostname, and port of PDS instance",
+					Value:   "https://public.api.bsky.app",
+					EnvVars: []string{"ATP_APPVIEW_HOST"},
+				},
+				&cli.StringFlag{
+					Name:     "http-address",
+					Usage:    "Specify the local IP/port to bind to",
+					Required: false,
+					Value:    ":8100",
+					EnvVars:  []string{"HTTP_ADDRESS"},
+				},
+				&cli.BoolFlag{
+					Name:     "debug",
+					Usage:    "Enable debug mode",
+					Value:    false,
+					Required: false,
+					EnvVars:  []string{"DEBUG"},
+				},
+			},
+		},
+	}
+	app.RunAndExitOnError()
+}

bskyweb/cmd/embedr/render.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"html/template"
+	"io"
+
+	"github.com/labstack/echo/v4"
+)
+
+type Template struct {
+	templates *template.Template
+}
+
+func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
+	return t.templates.ExecuteTemplate(w, name, data)
+}

bskyweb/cmd/embedr/server.go

@@ -0,0 +1,318 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"html/template"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	appbsky "github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+	"github.com/bluesky-social/indigo/util/cliutil"
+	"github.com/bluesky-social/indigo/xrpc"
+	"github.com/bluesky-social/social-app/bskyweb"
+
+	"github.com/klauspost/compress/gzhttp"
+	"github.com/klauspost/compress/gzip"
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+	"github.com/urfave/cli/v2"
+)
+
+type Server struct {
+	echo  *echo.Echo
+	httpd *http.Server
+	xrpcc *xrpc.Client
+}
+
+func serve(cctx *cli.Context) error {
+	debug := cctx.Bool("debug")
+	httpAddress := cctx.String("http-address")
+	appviewHost := cctx.String("appview-host")
+
+	// Echo
+	e := echo.New()
+
+	// create a new session (no auth)
+	xrpcc := &xrpc.Client{
+		Client: cliutil.NewHttpClient(),
+		Host:   appviewHost,
+	}
+
+	// httpd
+	var (
+		httpTimeout          = 2 * time.Minute
+		httpMaxHeaderBytes   = 2 * (1024 * 1024)
+		gzipMinSizeBytes     = 1024 * 2
+		gzipCompressionLevel = gzip.BestSpeed
+		gzipExceptMIMETypes  = []string{"image/png"}
+	)
+
+	// Wrap the server handler in a gzip handler to compress larger responses.
+	gzipHandler, err := gzhttp.NewWrapper(
+		gzhttp.MinSize(gzipMinSizeBytes),
+		gzhttp.CompressionLevel(gzipCompressionLevel),
+		gzhttp.ExceptContentTypes(gzipExceptMIMETypes),
+	)
+	if err != nil {
+		return err
+	}
+
+	//
+	// server
+	//
+	server := &Server{
+		echo:  e,
+		xrpcc: xrpcc,
+	}
+
+	// Create the HTTP server.
+	server.httpd = &http.Server{
+		Handler:        gzipHandler(server),
+		Addr:           httpAddress,
+		WriteTimeout:   httpTimeout,
+		ReadTimeout:    httpTimeout,
+		MaxHeaderBytes: httpMaxHeaderBytes,
+	}
+
+	e.HideBanner = true
+
+	tmpl := &Template{
+		templates: template.Must(template.ParseGlob("embed-templates/*.html")),
+	}
+	e.Renderer = tmpl
+	e.HTTPErrorHandler = server.errorHandler
+
+	e.IPExtractor = echo.ExtractIPFromXFFHeader()
+
+	// SECURITY: Do not modify without due consideration.
+	e.Use(middleware.SecureWithConfig(middleware.SecureConfig{
+		ContentTypeNosniff: "nosniff",
+		XFrameOptions:      "SAMEORIGIN",
+		HSTSMaxAge:         31536000, // 365 days
+		// TODO:
+		// ContentSecurityPolicy
+		// XSSProtection
+	}))
+	e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{
+		// Don't log requests for static content.
+		Skipper: func(c echo.Context) bool {
+			return strings.HasPrefix(c.Request().URL.Path, "/embed-static")
+		},
+	}))
+	e.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{
+		Skipper: middleware.DefaultSkipper,
+		Store: middleware.NewRateLimiterMemoryStoreWithConfig(
+			middleware.RateLimiterMemoryStoreConfig{
+				Rate:      10,              // requests per second
+				Burst:     30,              // allow bursts
+				ExpiresIn: 3 * time.Minute, // garbage collect entries older than 3 minutes
+			},
+		),
+		IdentifierExtractor: func(ctx echo.Context) (string, error) {
+			id := ctx.RealIP()
+			return id, nil
+		},
+		DenyHandler: func(c echo.Context, identifier string, err error) error {
+			return c.String(http.StatusTooManyRequests, "Your request has been rate limited. Please try again later. Contact support@bsky.app if you believe this was a mistake.\n")
+		},
+	}))
+
+	// redirect trailing slash to non-trailing slash.
+	// all of our current endpoints have no trailing slash.
+	e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{
+		RedirectCode: http.StatusFound,
+	}))
+
+	//
+	// configure routes
+	//
+	// static files
+	staticHandler := http.FileServer(func() http.FileSystem {
+		if debug {
+			log.Debugf("serving static file from the local file system")
+			return http.FS(os.DirFS("static"))
+		}
+		fsys, err := fs.Sub(bskyweb.StaticFS, "static")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return http.FS(fsys)
+	}())
+
+	e.GET("/robots.txt", echo.WrapHandler(staticHandler))
+	e.GET("/ips-v4", echo.WrapHandler(staticHandler))
+	e.GET("/ips-v6", echo.WrapHandler(staticHandler))
+	e.GET("/.well-known/*", echo.WrapHandler(staticHandler))
+	e.GET("/security.txt", func(c echo.Context) error {
+		return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt")
+	})
+	e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)), func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			path := c.Request().URL.Path
+			maxAge := 1 * (60 * 60) // default is 1 hour
+
+			// Cache javascript and images files for 1 week, which works because
+			// they're always versioned (e.g. /static/js/main.64c14927.js)
+			if strings.HasPrefix(path, "/static/js/") || strings.HasPrefix(path, "/static/images/") {
+				maxAge = 7 * (60 * 60 * 24) // 1 week
+			}
+
+			c.Response().Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge))
+			return next(c)
+		}
+	})
+
+	// actual routes
+	e.GET("/", server.WebHome)
+	e.GET("/about", server.WebGeneric)
+
+	// Start the server.
+	log.Infof("starting server address=%s", httpAddress)
+	go func() {
+		if err := server.httpd.ListenAndServe(); err != nil {
+			if !errors.Is(err, http.ErrServerClosed) {
+				log.Errorf("HTTP server shutting down unexpectedly: %s", err)
+			}
+		}
+	}()
+
+	// Wait for a signal to exit.
+	log.Info("registering OS exit signal handler")
+	quit := make(chan struct{})
+	exitSignals := make(chan os.Signal, 1)
+	signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-exitSignals
+		log.Infof("received OS exit signal: %s", sig)
+
+		// Shut down the HTTP server.
+		if err := server.Shutdown(); err != nil {
+			log.Errorf("HTTP server shutdown error: %s", err)
+		}
+
+		// Trigger the return that causes an exit.
+		close(quit)
+	}()
+	<-quit
+	log.Infof("graceful shutdown complete")
+	return nil
+}
+
+func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	srv.echo.ServeHTTP(rw, req)
+}
+
+func (srv *Server) Shutdown() error {
+	log.Info("shutting down")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	return srv.httpd.Shutdown(ctx)
+}
+
+func (srv *Server) errorHandler(err error, c echo.Context) {
+	code := http.StatusInternalServerError
+	if he, ok := err.(*echo.HTTPError); ok {
+		code = he.Code
+	}
+	c.Logger().Error(err)
+	data := map[string]interface{}{
+		"statusCode": code,
+	}
+	c.Render(code, "error.html", data)
+}
+
+// handler for endpoint that have no specific server-side handling
+func (srv *Server) WebGeneric(c echo.Context) error {
+	data := map[string]interface{}{}
+	return c.Render(http.StatusOK, "base.html", data)
+}
+
+func (srv *Server) WebHome(c echo.Context) error {
+	data := map[string]interface{}{}
+	return c.Render(http.StatusOK, "home.html", data)
+}
+
+func (srv *Server) WebPost(c echo.Context) error {
+	ctx := c.Request().Context()
+	data := map[string]interface{}{}
+
+	// sanity check arguments. don't 4xx, just let app handle if not expected format
+	rkeyParam := c.Param("rkey")
+	rkey, err := syntax.ParseRecordKey(rkeyParam)
+	if err != nil {
+		return c.Render(http.StatusOK, "post.html", data)
+	}
+	handleOrDIDParam := c.Param("handleOrDID")
+	handleOrDID, err := syntax.ParseAtIdentifier(handleOrDIDParam)
+	if err != nil {
+		return c.Render(http.StatusOK, "post.html", data)
+	}
+
+	identifier := handleOrDID.Normalize().String()
+
+	// requires two fetches: first fetch profile (!)
+	pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, identifier)
+	if err != nil {
+		log.Warnf("failed to fetch profile for: %s\t%v", identifier, err)
+		return c.Render(http.StatusOK, "post.html", data)
+	}
+	unauthedViewingOkay := true
+	for _, label := range pv.Labels {
+		if label.Src == pv.Did && label.Val == "!no-unauthenticated" {
+			unauthedViewingOkay = false
+		}
+	}
+
+	if !unauthedViewingOkay {
+		return c.Render(http.StatusOK, "post.html", data)
+	}
+	did := pv.Did
+	data["did"] = did
+
+	// then fetch the post thread (with extra context)
+	uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey)
+	tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 0, uri)
+	if err != nil {
+		log.Warnf("failed to fetch post: %s\t%v", uri, err)
+		return c.Render(http.StatusOK, "post.html", data)
+	}
+	req := c.Request()
+	postView := tpv.Thread.FeedDefs_ThreadViewPost.Post
+	data["postView"] = postView
+	data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path)
+	if postView.Embed != nil {
+		if postView.Embed.EmbedImages_View != nil {
+			var thumbUrls []string
+			for i := range postView.Embed.EmbedImages_View.Images {
+				thumbUrls = append(thumbUrls, postView.Embed.EmbedImages_View.Images[i].Thumb)
+			}
+			data["imgThumbUrls"] = thumbUrls
+		} else if postView.Embed.EmbedRecordWithMedia_View != nil && postView.Embed.EmbedRecordWithMedia_View.Media != nil && postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View != nil {
+			var thumbUrls []string
+			for i := range postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images {
+				thumbUrls = append(thumbUrls, postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images[i].Thumb)
+			}
+			data["imgThumbUrls"] = thumbUrls
+		}
+	}
+
+	if postView.Record != nil {
+		postRecord, ok := postView.Record.Val.(*appbsky.FeedPost)
+		if ok {
+			_ = postRecord
+			data["postText"] = "" // XXX
+		}
+	}
+
+	return c.Render(http.StatusOK, "post.html", data)
+}

bskyweb/embed-static/.well-known/security.txt

@@ -0,0 +1,4 @@
+Contact: mailto:security@bsky.app
+Preferred-Languages: en
+Canonical: https://bsky.app/.well-known/security.txt
+Acknowledgements: https://github.com/bluesky-social/atproto/blob/main/CONTRIBUTORS.md

bskyweb/embed-static/ips-v4

@@ -0,0 +1,30 @@
+13.59.225.103/32
+3.18.47.21/32
+18.191.104.94/32
+3.129.134.255/32
+3.129.237.113/32
+3.138.56.230/32
+44.218.10.163/32
+54.89.116.251/32
+44.217.166.202/32
+54.208.221.149/32
+54.166.110.54/32
+54.208.146.65/32
+3.129.234.15/32
+3.138.168.48/32
+3.23.53.192/32
+52.14.89.53/32
+3.18.126.246/32
+3.136.69.4/32
+3.22.137.152/32
+3.132.247.113/32
+3.141.186.104/32
+18.222.43.214/32
+3.14.35.197/32
+3.23.182.70/32
+18.224.144.69/32
+3.129.98.29/32
+3.130.134.20/32
+3.17.197.213/32
+18.223.234.21/32
+3.20.248.177/32

bskyweb/embed-static/robots.txt

@@ -0,0 +1,9 @@
+# Hello Friends!
+# If you are considering bulk or automated crawling, you may want to look in
+# to our protocol (API), including a firehose of updates. See: https://atproto.com/
+
+# By default, may crawl anything on this domain. HTTP 429 ("backoff") status
+# codes are used for rate-limiting. Up to a handful concurrent requests should
+# be ok.
+User-Agent: *
+Allow: /

bskyweb/embed-templates/error.html

@@ -0,0 +1 @@
+placeholder!

bskyweb/embed-templates/home.html

@@ -0,0 +1 @@
+embed homepage