From 11106c1ecb2d7c8fc499511e692bc446f6bf7e1a Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Thu, 11 Apr 2024 19:38:37 -0700 Subject: [PATCH] skeleton of embedr service, based on bskyweb --- bskyweb/cmd/embedr/.gitignore | 1 + bskyweb/cmd/embedr/main.go | 60 ++++ bskyweb/cmd/embedr/render.go | 16 + bskyweb/cmd/embedr/server.go | 318 ++++++++++++++++++ bskyweb/embed-static/.well-known/security.txt | 4 + bskyweb/embed-static/favicon-16x16.png | Bin 0 -> 1731 bytes bskyweb/embed-static/favicon-32x32.png | Bin 0 -> 2240 bytes bskyweb/embed-static/favicon.png | Bin 0 -> 1412 bytes bskyweb/embed-static/ips-v4 | 30 ++ bskyweb/embed-static/ips-v6 | 0 bskyweb/embed-static/robots.txt | 9 + bskyweb/embed-templates/error.html | 1 + bskyweb/embed-templates/home.html | 1 + 13 files changed, 440 insertions(+) create mode 100644 bskyweb/cmd/embedr/.gitignore create mode 100644 bskyweb/cmd/embedr/main.go create mode 100644 bskyweb/cmd/embedr/render.go create mode 100644 bskyweb/cmd/embedr/server.go create mode 100644 bskyweb/embed-static/.well-known/security.txt create mode 100644 bskyweb/embed-static/favicon-16x16.png create mode 100644 bskyweb/embed-static/favicon-32x32.png create mode 100644 bskyweb/embed-static/favicon.png create mode 100644 bskyweb/embed-static/ips-v4 create mode 100644 bskyweb/embed-static/ips-v6 create mode 100644 bskyweb/embed-static/robots.txt create mode 100644 bskyweb/embed-templates/error.html create mode 100644 bskyweb/embed-templates/home.html diff --git a/bskyweb/cmd/embedr/.gitignore b/bskyweb/cmd/embedr/.gitignore new file mode 100644 index 0000000000..c810652a10 --- /dev/null +++ b/bskyweb/cmd/embedr/.gitignore @@ -0,0 +1 @@ +/bskyweb diff --git a/bskyweb/cmd/embedr/main.go b/bskyweb/cmd/embedr/main.go new file mode 100644 index 0000000000..9f75ed69af --- /dev/null +++ b/bskyweb/cmd/embedr/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + + _ "github.com/joho/godotenv/autoload" + + logging "github.com/ipfs/go-log" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("embedr") + +func init() { + logging.SetAllLoggers(logging.LevelDebug) + //logging.SetAllLoggers(logging.LevelWarn) +} + +func main() { + run(os.Args) +} + +func run(args []string) { + + app := cli.App{ + Name: "embedr", + Usage: "web server for embed.bsky.app post embeds", + } + + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the server", + Action: serve, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "appview-host", + Usage: "method, hostname, and port of PDS instance", + Value: "https://public.api.bsky.app", + EnvVars: []string{"ATP_APPVIEW_HOST"}, + }, + &cli.StringFlag{ + Name: "http-address", + Usage: "Specify the local IP/port to bind to", + Required: false, + Value: ":8100", + EnvVars: []string{"HTTP_ADDRESS"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug mode", + Value: false, + Required: false, + EnvVars: []string{"DEBUG"}, + }, + }, + }, + } + app.RunAndExitOnError() +} diff --git a/bskyweb/cmd/embedr/render.go b/bskyweb/cmd/embedr/render.go new file mode 100644 index 0000000000..cc8f0759a0 --- /dev/null +++ b/bskyweb/cmd/embedr/render.go @@ -0,0 +1,16 @@ +package main + +import ( + "html/template" + "io" + + "github.com/labstack/echo/v4" +) + +type Template struct { + templates *template.Template +} + +func (t *Template) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + return t.templates.ExecuteTemplate(w, name, data) +} diff --git a/bskyweb/cmd/embedr/server.go b/bskyweb/cmd/embedr/server.go new file mode 100644 index 0000000000..2c519106dc --- /dev/null +++ b/bskyweb/cmd/embedr/server.go @@ -0,0 +1,318 @@ +package main + +import ( + "context" + "errors" + "fmt" + "html/template" + "io/fs" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + "github.com/bluesky-social/social-app/bskyweb" + + "github.com/klauspost/compress/gzhttp" + "github.com/klauspost/compress/gzip" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/urfave/cli/v2" +) + +type Server struct { + echo *echo.Echo + httpd *http.Server + xrpcc *xrpc.Client +} + +func serve(cctx *cli.Context) error { + debug := cctx.Bool("debug") + httpAddress := cctx.String("http-address") + appviewHost := cctx.String("appview-host") + + // Echo + e := echo.New() + + // create a new session (no auth) + xrpcc := &xrpc.Client{ + Client: cliutil.NewHttpClient(), + Host: appviewHost, + } + + // httpd + var ( + httpTimeout = 2 * time.Minute + httpMaxHeaderBytes = 2 * (1024 * 1024) + gzipMinSizeBytes = 1024 * 2 + gzipCompressionLevel = gzip.BestSpeed + gzipExceptMIMETypes = []string{"image/png"} + ) + + // Wrap the server handler in a gzip handler to compress larger responses. + gzipHandler, err := gzhttp.NewWrapper( + gzhttp.MinSize(gzipMinSizeBytes), + gzhttp.CompressionLevel(gzipCompressionLevel), + gzhttp.ExceptContentTypes(gzipExceptMIMETypes), + ) + if err != nil { + return err + } + + // + // server + // + server := &Server{ + echo: e, + xrpcc: xrpcc, + } + + // Create the HTTP server. + server.httpd = &http.Server{ + Handler: gzipHandler(server), + Addr: httpAddress, + WriteTimeout: httpTimeout, + ReadTimeout: httpTimeout, + MaxHeaderBytes: httpMaxHeaderBytes, + } + + e.HideBanner = true + + tmpl := &Template{ + templates: template.Must(template.ParseGlob("embed-templates/*.html")), + } + e.Renderer = tmpl + e.HTTPErrorHandler = server.errorHandler + + e.IPExtractor = echo.ExtractIPFromXFFHeader() + + // SECURITY: Do not modify without due consideration. + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + XFrameOptions: "SAMEORIGIN", + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ + // Don't log requests for static content. + Skipper: func(c echo.Context) bool { + return strings.HasPrefix(c.Request().URL.Path, "/embed-static") + }, + })) + e.Use(middleware.RateLimiterWithConfig(middleware.RateLimiterConfig{ + Skipper: middleware.DefaultSkipper, + Store: middleware.NewRateLimiterMemoryStoreWithConfig( + middleware.RateLimiterMemoryStoreConfig{ + Rate: 10, // requests per second + Burst: 30, // allow bursts + ExpiresIn: 3 * time.Minute, // garbage collect entries older than 3 minutes + }, + ), + IdentifierExtractor: func(ctx echo.Context) (string, error) { + id := ctx.RealIP() + return id, nil + }, + DenyHandler: func(c echo.Context, identifier string, err error) error { + return c.String(http.StatusTooManyRequests, "Your request has been rate limited. Please try again later. Contact support@bsky.app if you believe this was a mistake.\n") + }, + })) + + // redirect trailing slash to non-trailing slash. + // all of our current endpoints have no trailing slash. + e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ + RedirectCode: http.StatusFound, + })) + + // + // configure routes + // + // static files + staticHandler := http.FileServer(func() http.FileSystem { + if debug { + log.Debugf("serving static file from the local file system") + return http.FS(os.DirFS("static")) + } + fsys, err := fs.Sub(bskyweb.StaticFS, "static") + if err != nil { + log.Fatal(err) + } + return http.FS(fsys) + }()) + + e.GET("/robots.txt", echo.WrapHandler(staticHandler)) + e.GET("/ips-v4", echo.WrapHandler(staticHandler)) + e.GET("/ips-v6", echo.WrapHandler(staticHandler)) + e.GET("/.well-known/*", echo.WrapHandler(staticHandler)) + e.GET("/security.txt", func(c echo.Context) error { + return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt") + }) + e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)), func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + path := c.Request().URL.Path + maxAge := 1 * (60 * 60) // default is 1 hour + + // Cache javascript and images files for 1 week, which works because + // they're always versioned (e.g. /static/js/main.64c14927.js) + if strings.HasPrefix(path, "/static/js/") || strings.HasPrefix(path, "/static/images/") { + maxAge = 7 * (60 * 60 * 24) // 1 week + } + + c.Response().Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", maxAge)) + return next(c) + } + }) + + // actual routes + e.GET("/", server.WebHome) + e.GET("/about", server.WebGeneric) + + // Start the server. + log.Infof("starting server address=%s", httpAddress) + go func() { + if err := server.httpd.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + log.Errorf("HTTP server shutting down unexpectedly: %s", err) + } + } + }() + + // Wait for a signal to exit. + log.Info("registering OS exit signal handler") + quit := make(chan struct{}) + exitSignals := make(chan os.Signal, 1) + signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-exitSignals + log.Infof("received OS exit signal: %s", sig) + + // Shut down the HTTP server. + if err := server.Shutdown(); err != nil { + log.Errorf("HTTP server shutdown error: %s", err) + } + + // Trigger the return that causes an exit. + close(quit) + }() + <-quit + log.Infof("graceful shutdown complete") + return nil +} + +func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + srv.echo.ServeHTTP(rw, req) +} + +func (srv *Server) Shutdown() error { + log.Info("shutting down") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.httpd.Shutdown(ctx) +} + +func (srv *Server) errorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + } + c.Logger().Error(err) + data := map[string]interface{}{ + "statusCode": code, + } + c.Render(code, "error.html", data) +} + +// handler for endpoint that have no specific server-side handling +func (srv *Server) WebGeneric(c echo.Context) error { + data := map[string]interface{}{} + return c.Render(http.StatusOK, "base.html", data) +} + +func (srv *Server) WebHome(c echo.Context) error { + data := map[string]interface{}{} + return c.Render(http.StatusOK, "home.html", data) +} + +func (srv *Server) WebPost(c echo.Context) error { + ctx := c.Request().Context() + data := map[string]interface{}{} + + // sanity check arguments. don't 4xx, just let app handle if not expected format + rkeyParam := c.Param("rkey") + rkey, err := syntax.ParseRecordKey(rkeyParam) + if err != nil { + return c.Render(http.StatusOK, "post.html", data) + } + handleOrDIDParam := c.Param("handleOrDID") + handleOrDID, err := syntax.ParseAtIdentifier(handleOrDIDParam) + if err != nil { + return c.Render(http.StatusOK, "post.html", data) + } + + identifier := handleOrDID.Normalize().String() + + // requires two fetches: first fetch profile (!) + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, identifier) + if err != nil { + log.Warnf("failed to fetch profile for: %s\t%v", identifier, err) + return c.Render(http.StatusOK, "post.html", data) + } + unauthedViewingOkay := true + for _, label := range pv.Labels { + if label.Src == pv.Did && label.Val == "!no-unauthenticated" { + unauthedViewingOkay = false + } + } + + if !unauthedViewingOkay { + return c.Render(http.StatusOK, "post.html", data) + } + did := pv.Did + data["did"] = did + + // then fetch the post thread (with extra context) + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 0, uri) + if err != nil { + log.Warnf("failed to fetch post: %s\t%v", uri, err) + return c.Render(http.StatusOK, "post.html", data) + } + req := c.Request() + postView := tpv.Thread.FeedDefs_ThreadViewPost.Post + data["postView"] = postView + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + if postView.Embed != nil { + if postView.Embed.EmbedImages_View != nil { + var thumbUrls []string + for i := range postView.Embed.EmbedImages_View.Images { + thumbUrls = append(thumbUrls, postView.Embed.EmbedImages_View.Images[i].Thumb) + } + data["imgThumbUrls"] = thumbUrls + } else if postView.Embed.EmbedRecordWithMedia_View != nil && postView.Embed.EmbedRecordWithMedia_View.Media != nil && postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View != nil { + var thumbUrls []string + for i := range postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images { + thumbUrls = append(thumbUrls, postView.Embed.EmbedRecordWithMedia_View.Media.EmbedImages_View.Images[i].Thumb) + } + data["imgThumbUrls"] = thumbUrls + } + } + + if postView.Record != nil { + postRecord, ok := postView.Record.Val.(*appbsky.FeedPost) + if ok { + _ = postRecord + data["postText"] = "" // XXX + } + } + + return c.Render(http.StatusOK, "post.html", data) +} diff --git a/bskyweb/embed-static/.well-known/security.txt b/bskyweb/embed-static/.well-known/security.txt new file mode 100644 index 0000000000..8173cb72d6 --- /dev/null +++ b/bskyweb/embed-static/.well-known/security.txt @@ -0,0 +1,4 @@ +Contact: mailto:security@bsky.app +Preferred-Languages: en +Canonical: https://bsky.app/.well-known/security.txt +Acknowledgements: https://github.com/bluesky-social/atproto/blob/main/CONTRIBUTORS.md diff --git a/bskyweb/embed-static/favicon-16x16.png b/bskyweb/embed-static/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..ea256e0569cee14f07f89970315bab56559207d3 GIT binary patch literal 1731 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|j-^I;ruq6Z zXaU(A3~Y=-49p-UK*+!-#lQ+?GcbfPO2gT4j2ciiOh7e;3_y}W6o}K>GZ|Q*>T7^B z2zUT7&?E>QkXezMlbcwQU!)LFl&@f{XR2oilw+B|0yaYg$lkPo5n=xVCb)S53z!jX zpgIO410!QALnA9ALj^+%D-&}oQ&Wb`59aL$N^ur=L>4nJa0`PlBg3pY5)2H?8!|&8 zN+NuHtdjF{^%7I^lT!66atlBvG1ydC0hzg}C5Z|ZxjA{oRu#5NU=>zCHb_`sNdc^+ zB->Ug!Z$#{Ilm}X!A#FU&p^qJOF==wrYI%ND#*nRsvXF)RmvzSDX`MlFE20GD>v55 zFG|-pw6wI;H!#vSGSUUA&@HaaD@m--%_~-h7y>iLCAB!YD6^m>Ge1uOWNuP#1_oB9X1WFzK!%MzhFUapoQqO{CSaHX z%|ytiAgRP=Mt)I9etwP}wp0W1QrG6AUiIgPFQZV<5D)u zIS4F2Zh5*mhFAzL4Yth|2^87;{OpdVU7d!~le~geN?lBrTs&u=p!$PZWvWsKo1pll z#K4Uky@Yd)xLx#8>{V%%??}*Dv&BFu%i!4AUyQqEmhbt#bMx<*D{AF-?<~LHD}H{i z+E{6}$(M5l`L$P-C`D$C! zK9PmOa!jOzDTOO{;VZG<~Y+oJ;ue zMA6mKuNM7qH!<84=Bn(oVD<**$#;`h*Ds1c8Kk1~;DDP_vzt@%`R7j>AGNz~d)xV% z<=vJ_?ibtc)-|;VxO_^C<_h~Dr9QJhe Kb6Mw<&;$S(wmP-| literal 0 HcmV?d00001 diff --git a/bskyweb/embed-static/favicon-32x32.png b/bskyweb/embed-static/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..a5ca7eed1e24b9554e417be0b81b5d0cf34bcdd5 GIT binary patch literal 2240 zcmaJ?3piA17(N*HW;MDfl4&Xlb2p7SjZutSx#iZ7Y(obVW|)~WDbY%)R4UX;#t^Bn zNEdA>o7*GFZq(C7Ek%SbWTLk0IYUvaJ@fqk|DFGP-}n3e|9sywd)-|f&|3Oh005w! z9N8Z5ibaNo8a#_)V&1`vlE}ls1}JWt*8{)QLL8@H7Z<=3wlx4XC0#%TfxtgNNgq&E z*Z|<8WboAvP%;_ED8m>s0MEcIcm(g^0jP2Oc6db|M|i+WW2-w)0)N3ID}6tgdz7Xk4_$oZ&noGBes5;;F2PI62l!gmk4 zz?Ha>E9SDfVu*#5Kb}m%Q^;O$`P1ki*#b|p1d*%#5B579TOkB1OrfG^qRA?gkW*<0 zgkAzcIBTk|9?L?(A4iP~Fo{S_Ppa{;22$+89mLXtnP0k*r3IZn;sf{YeJA!xFOFvC z+I68`A=9%)=msl~8rRWGw_I zSRB))-mBdB(76BNDx(#B8}hY-3`QEf%$@n17I(G-+8OFF_YJ#WbeA|xQ0kx01*VV{M z>=?BT4CbXJ$<;KyQkM$n+^B+LXyszFknZG`^{#hRIC*xJ36Iyk*Tm#*wa@Kp4;HAJ zV$7Y^GX^iVrJ=w}{rtAFMa;nH)msU318Zw?KVnF0X0@Z6Qnt@od_#`WjGKSKE$69c zW4E5X_h8cj9Y%Ah-IdtN0LUxlQ{|KT{VM->rp#D@$+bkcbBAb_VTUSq z-jwHtHD)<{V*1=6rmrZ*Uh~pg8MUK!m9d)ZlZ7A5B-iPG z3^CGs41d1INR6K}D4Onk)OT3j{Dq@FahJgdYQdf+z%aFtY8sai-*p6heD;>ekJ{o* z6D~|ELdn8RR=hUOQhFJr_X#pA=&`o(t&BVVqNCgRUd9e@sg&aX?z3A$RhrR5Wm>A* z2#y+;H?T;hwIzHg>dpHR;a}#Bu|Wyj8T;yV-EB&x(K)TNbDn1q0`S*g_14TYOvo%p zyG31R5rzgaQlrdXVnl literal 0 HcmV?d00001 diff --git a/bskyweb/embed-static/favicon.png b/bskyweb/embed-static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..ddf55f4c815e6aa50cdb361c40dbf9e6ecfbe64a GIT binary patch literal 1412 zcmV-~1$+95P)~#+)E<0^kIYWrpF!JOY_r-~@3_fH^_P3Gkd?lzl#wZ9rYA zYRQfucI&HK*ulT|{4Cr0s#0}#)xX;S3l=Q6&){ey{Fio6(=HHnvaf~+G*n8x{rDU& zU=Dfmin^NMDG_z~V~1ZMzlI9*>8HJ=zU*ThuzXCfd8b{=xgtL3V06%|vvNdR8flXs zW_Q{?{BZl9_kwK<*-QN7-$6MdQn(gB#|UPS@Q79s>9x)9Jb}hFr8I0pc{=Q@|Ce9U z3z#uX69&30YrLS9-o(d8`jWR-p(+R$gt&dz(Gd11#D{cN#qb>5wCflz@CBvma1-jl z9U<3d<1cF(FLV?%qjOhIFzubV@A@%n)uO2b1aZQp+z_;cF&>&5FDO7KZ8(%0YQqU* z;Q_u9p3r`V?4PD`!Q%7fHAlwbpj7h;0QC+ zxi&sSA?UB*9vPnAfFsPgJQ&YTkVzWL`K77@u`-j~u3YRKUS zZ9;%M90e!v$m9t*_hXax;7~lhNR=kuXG##RbuwB<@APm#DjK|0NT?27(zF6?nM+J4EX79j6UAb1ZAQFJeJ2ZK*Tma%YSjh0W!-& zgd4x86DH1U;-o#>&dFO~$i{C>n#M!hQQ^yE7ZvWzu_IzurB37hpf_y?s{=-aB26JZ z%{oZ0B83BD8Rit>x#Ty~D$fz69FWJ%{&4aVs5B~gNa=D`x5{Vw4cc)|Y7i$fqUv%M z`(8dKn!jIyVb$5{$_@~6OHa1o?jwSoKo?r6gE~|7l^h_3Cu9njF3ch|)-05s8x)-` z(KOqMQo3fL(h8PR5ipk-qC~?t9{LJ8!NLg06xuG#8+^~KITVU4)(BWRyk6(djn`$I*-!=%SLVfE#2xAb@$%P*5YM@r>1ysGg+<+Z|TBvh2n)6#Ui z?C$zn$;@$;9E8PBT&rq)dTTWi>0BkcSM;R#shskS)mrmYCfX#S(pTzPQQbfN_=Xrq zXuZqA=wC3(7dSdtusqjV4dVS~f4R#!`j71VVrazu(K#%s{;d9iYr%pA^TBu8qIv!L S#kWBK0000