diff --git a/bskyweb/cmd/embedr/.gitignore b/bskyweb/cmd/embedr/.gitignore new file mode 100644 index 0000000000..c810652a10 --- /dev/null +++ b/bskyweb/cmd/embedr/.gitignore @@ -0,0 +1 @@ +/bskyweb diff --git a/bskyweb/cmd/embedr/main.go b/bskyweb/cmd/embedr/main.go new file mode 100644 index 0000000000..9f75ed69af --- /dev/null +++ b/bskyweb/cmd/embedr/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + + _ "github.com/joho/godotenv/autoload" + + logging "github.com/ipfs/go-log" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("embedr") + +func init() { + logging.SetAllLoggers(logging.LevelDebug) + //logging.SetAllLoggers(logging.LevelWarn) +} + +func main() { + run(os.Args) +} + +func run(args []string) { + + app := cli.App{ + Name: "embedr", + Usage: "web server for embed.bsky.app post embeds", + } + + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the server", + Action: serve, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "appview-host", + Usage: "method, hostname, and port of PDS instance", + Value: "https://public.api.bsky.app", + EnvVars: []string{"ATP_APPVIEW_HOST"}, + }, + &cli.StringFlag{ + Name: "http-address", + Usage: "Specify the local IP/port to bind to", + Required: false, + Value: ":8100", + EnvVars: []string{"HTTP_ADDRESS"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug mode", + Value: false, + Required: false, + EnvVars: []string{"DEBUG"}, + }, + }, + }, + } + app.RunAndExitOnError() +} diff --git a/bskyweb/cmd/embedr/render.go b/bskyweb/cmd/embedr/render.go new file mode 100644 index 0000000000..cc8f0759a0 --- /dev/null +++ b/bskyweb/cmd/embedr/render.go @@ -0,0 +1,16 @@ +package main + +import ( + "html/template" + "io" + + "github.com/labstack/echo/v4" +) + +type Template struct { + templates *template.Template +} + +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) +} diff --git a/bskyweb/cmd/embedr/server.go b/bskyweb/cmd/embedr/server.go new file mode 100644 index 0000000000..2c519106dc --- /dev/null +++ b/bskyweb/cmd/embedr/server.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "errors" + "fmt" + "html/template" + "io/fs" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + "github.com/bluesky-social/social-app/bskyweb" + + "github.com/klauspost/compress/gzhttp" + "github.com/klauspost/compress/gzip" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/urfave/cli/v2" +) + +type Server struct { + echo *echo.Echo + httpd *http.Server + xrpcc *xrpc.Client +} + +func serve(cctx *cli.Context) error { + debug := cctx.Bool("debug") + httpAddress := cctx.String("http-address") + appviewHost := cctx.String("appview-host") + + // Echo + e := echo.New() + + // create a new session (no auth) + xrpcc := &xrpc.Client{ + Client: cliutil.NewHttpClient(), + Host: appviewHost, + } + + // httpd + var ( + httpTimeout = 2 * time.Minute + httpMaxHeaderBytes = 2 * (1024 * 1024) + gzipMinSizeBytes = 1024 * 2 + gzipCompressionLevel = gzip.BestSpeed + gzipExceptMIMETypes = []string{"image/png"} + ) + + // Wrap the server handler in a gzip handler to compress larger responses. + gzipHandler, err := gzhttp.NewWrapper( + gzhttp.MinSize(gzipMinSizeBytes), + gzhttp.CompressionLevel(gzipCompressionLevel), + gzhttp.ExceptContentTypes(gzipExceptMIMETypes), + ) + if err != nil { + return err + } + + // + // server + // + server := &Server{ + echo: e, + xrpcc: xrpcc, + } + + // Create the HTTP server. + server.httpd = &http.Server{ + Handler: gzipHandler(server), + Addr: httpAddress, + WriteTimeout: httpTimeout, + ReadTimeout: httpTimeout, + MaxHeaderBytes: httpMaxHeaderBytes, + } + + e.HideBanner = true + + tmpl := &Template{ + templates: template.Must(template.ParseGlob("embed-templates/*.html")), + } + e.Renderer = tmpl + e.HTTPErrorHandler = server.errorHandler + + e.IPExtractor = echo.ExtractIPFromXFFHeader() + + // SECURITY: Do not modify without due consideration. + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + XFrameOptions: "SAMEORIGIN", + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ + // Don't log requests for static content. + Skipper: func(c echo.Context) bool { + return strings.HasPrefix(c.Request().URL.Path, "/embed-static") + }, + })) + e.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{ + Skipper: middleware.DefaultSkipper, + Store: middleware.NewRateLimiterMemoryStoreWithConfig( + middleware.RateLimiterMemoryStoreConfig{ + Rate: 10, // requests per second + Burst: 30, // allow bursts + ExpiresIn: 3 * time.Minute, // garbage collect entries older than 3 minutes + }, + ), + IdentifierExtractor: func(ctx echo.Context) (string, error) { + id := ctx.RealIP() + return id, nil + }, + DenyHandler: func(c echo.Context, identifier string, err error) error { + return c.String(http.StatusTooManyRequests, "Your request has been rate limited. Please try again later. Contact support@bsky.app if you believe this was a mistake.\n") + }, + })) + + // redirect trailing slash to non-trailing slash. + // all of our current endpoints have no trailing slash. + e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ + RedirectCode: http.StatusFound, + })) + + // + // configure routes + // + // static files + staticHandler := http.FileServer(func() http.FileSystem { + if debug { + log.Debugf("serving static file from the local file system") + return http.FS(os.DirFS("static")) + } + fsys, err := fs.Sub(bskyweb.StaticFS, "static") + if err != nil { + log.Fatal(err) + } + return http.FS(fsys) + }()) + + e.GET("/robots.txt", echo.WrapHandler(staticHandler)) + e.GET("/ips-v4", echo.WrapHandler(staticHandler)) + e.GET("/ips-v6", echo.WrapHandler(staticHandler)) + e.GET("/.well-known/*", echo.WrapHandler(staticHandler)) + e.GET("/security.txt", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt") + }) + e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)), func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + path := c.Request().URL.Path + maxAge := 1 * (60 * 60) // default is 1 hour + + // Cache javascript and images files for 1 week, which works because + // they're always versioned (e.g. /static/js/main.64c14927.js) + if strings.HasPrefix(path, "/static/js/") || strings.HasPrefix(path, "/static/images/") { + maxAge = 7 * (60 * 60 * 24) // 1 week + } + + c.Response().Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) + return next(c) + } + }) + + // actual routes + e.GET("/", server.WebHome) + e.GET("/about", server.WebGeneric) + + // Start the server. + log.Infof("starting server address=%s", httpAddress) + go func() { + if err := server.httpd.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Errorf("HTTP server shutting down unexpectedly: %s", err) + } + } + }() + + // Wait for a signal to exit. + log.Info("registering OS exit signal handler") + quit := make(chan struct{}) + exitSignals := make(chan os.Signal, 1) + signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-exitSignals + log.Infof("received OS exit signal: %s", sig) + + // Shut down the HTTP server. + if err := server.Shutdown(); err != nil { + log.Errorf("HTTP server shutdown error: %s", err) + } + + // Trigger the return that causes an exit. + close(quit) + }() + <-quit + log.Infof("graceful shutdown complete") + return nil +} + +func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + srv.echo.ServeHTTP(rw, req) +} + +func (srv *Server) Shutdown() error { + log.Info("shutting down") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.httpd.Shutdown(ctx) +} + +func (srv *Server) errorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + } + c.Logger().Error(err) + data := map[string]interface{}{ + "statusCode": code, + } + c.Render(code, "error.html", data) +} + +// handler for endpoint that have no specific server-side handling +func (srv *Server) WebGeneric(c echo.Context) error { + data := map[string]interface{}{} + return c.Render(http.StatusOK, "base.html", data) +} + +func (srv *Server) WebHome(c echo.Context) error { + data := map[string]interface{}{} + return c.Render(http.StatusOK, "home.html", data) +} + +func (srv *Server) WebPost(c echo.Context) error { + ctx := c.Request().Context() + data := map[string]interface{}{} + + // sanity check arguments. don't 4xx, just let app handle if not expected format + rkeyParam := c.Param("rkey") + rkey, err := syntax.ParseRecordKey(rkeyParam) + if err != nil { + return c.Render(http.StatusOK, "post.html", data) + } + handleOrDIDParam := c.Param("handleOrDID") + handleOrDID, err := syntax.ParseAtIdentifier(handleOrDIDParam) + if err != nil { + return c.Render(http.StatusOK, "post.html", data) + } + + identifier := handleOrDID.Normalize().String() + + // requires two fetches: first fetch profile (!) + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, identifier) + if err != nil { + log.Warnf("failed to fetch profile for: %s\t%v", identifier, err) + return c.Render(http.StatusOK, "post.html", data) + } + unauthedViewingOkay := true + for _, label := range pv.Labels { + if label.Src == pv.Did && label.Val == "!no-unauthenticated" { + unauthedViewingOkay = false + } + } + + if !unauthedViewingOkay { + return c.Render(http.StatusOK, "post.html", data) + } + did := pv.Did + data["did"] = did + + // then fetch the post thread (with extra context) + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 0, uri) + if err != nil { + log.Warnf("failed to fetch post: %s\t%v", uri, err) + return c.Render(http.StatusOK, "post.html", data) + } + req := c.Request() + postView := tpv.Thread.FeedDefs_ThreadViewPost.Post + data["postView"] = postView + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + if postView.Embed != nil { + if postView.Embed.EmbedImages_View != nil { + var thumbUrls []string + for i := range postView.Embed.EmbedImages_View.Images { + thumbUrls = append(thumbUrls, postView.Embed.EmbedImages_View.Images[i].Thumb) + } + data["imgThumbUrls"] = thumbUrls + } else if postView.Embed.EmbedRecordWithMedia_View != nil && postView.Embed.EmbedRecordWithMedia_View.Media != nil && postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View != nil { + var thumbUrls []string + for i := range postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images { + thumbUrls = append(thumbUrls, postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images[i].Thumb) + } + data["imgThumbUrls"] = thumbUrls + } + } + + if postView.Record != nil { + postRecord, ok := postView.Record.Val.(*appbsky.FeedPost) + if ok { + _ = postRecord + data["postText"] = "" // XXX + } + } + + return c.Render(http.StatusOK, "post.html", data) +} diff --git a/bskyweb/embed-static/.well-known/security.txt b/bskyweb/embed-static/.well-known/security.txt new file mode 100644 index 0000000000..8173cb72d6 --- /dev/null +++ b/bskyweb/embed-static/.well-known/security.txt @@ -0,0 +1,4 @@ +Contact: mailto:security@bsky.app +Preferred-Languages: en +Canonical: https://bsky.app/.well-known/security.txt +Acknowledgements: https://github.com/bluesky-social/atproto/blob/main/CONTRIBUTORS.md diff --git a/bskyweb/embed-static/favicon-16x16.png b/bskyweb/embed-static/favicon-16x16.png new file mode 100644 index 0000000000..ea256e0569 Binary files /dev/null and b/bskyweb/embed-static/favicon-16x16.png differ diff --git a/bskyweb/embed-static/favicon-32x32.png b/bskyweb/embed-static/favicon-32x32.png new file mode 100644 index 0000000000..a5ca7eed1e Binary files /dev/null and b/bskyweb/embed-static/favicon-32x32.png differ diff --git a/bskyweb/embed-static/favicon.png b/bskyweb/embed-static/favicon.png new file mode 100644 index 0000000000..ddf55f4c81 Binary files /dev/null and b/bskyweb/embed-static/favicon.png differ diff --git a/bskyweb/embed-static/ips-v4 b/bskyweb/embed-static/ips-v4 new file mode 100644 index 0000000000..087996ef9a --- /dev/null +++ b/bskyweb/embed-static/ips-v4 @@ -0,0 +1,30 @@ +13.59.225.103/32 +3.18.47.21/32 +18.191.104.94/32 +3.129.134.255/32 +3.129.237.113/32 +3.138.56.230/32 +44.218.10.163/32 +54.89.116.251/32 +44.217.166.202/32 +54.208.221.149/32 +54.166.110.54/32 +54.208.146.65/32 +3.129.234.15/32 +3.138.168.48/32 +3.23.53.192/32 +52.14.89.53/32 +3.18.126.246/32 +3.136.69.4/32 +3.22.137.152/32 +3.132.247.113/32 +3.141.186.104/32 +18.222.43.214/32 +3.14.35.197/32 +3.23.182.70/32 +18.224.144.69/32 +3.129.98.29/32 +3.130.134.20/32 +3.17.197.213/32 +18.223.234.21/32 +3.20.248.177/32 diff --git a/bskyweb/embed-static/ips-v6 b/bskyweb/embed-static/ips-v6 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bskyweb/embed-static/robots.txt b/bskyweb/embed-static/robots.txt new file mode 100644 index 0000000000..4f8510d18d --- /dev/null +++ b/bskyweb/embed-static/robots.txt @@ -0,0 +1,9 @@ +# Hello Friends! +# If you are considering bulk or automated crawling, you may want to look in +# to our protocol (API), including a firehose of updates. See: https://atproto.com/ + +# By default, may crawl anything on this domain. HTTP 429 ("backoff") status +# codes are used for rate-limiting. Up to a handful concurrent requests should +# be ok. +User-Agent: * +Allow: / diff --git a/bskyweb/embed-templates/error.html b/bskyweb/embed-templates/error.html new file mode 100644 index 0000000000..5aa04c83bf --- /dev/null +++ b/bskyweb/embed-templates/error.html @@ -0,0 +1 @@ +placeholder! diff --git a/bskyweb/embed-templates/home.html b/bskyweb/embed-templates/home.html new file mode 100644 index 0000000000..0ff746b56a --- /dev/null +++ b/bskyweb/embed-templates/home.html @@ -0,0 +1 @@ +embed homepage