diff --git a/bskyweb/cmd/bskyweb/server.go b/bskyweb/cmd/bskyweb/server.go index ec5261dee8..790f211eed 100644 --- a/bskyweb/cmd/bskyweb/server.go +++ b/bskyweb/cmd/bskyweb/server.go @@ -9,7 +9,9 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" + "log/slog" "net/http" "net/netip" "net/url" @@ -40,6 +42,11 @@ type Server struct { cfg *Config ipccClient http.Client + + // sitemapClient is used for fetching sitemaps from the appview. It has + // DisableCompression set to true so that gzipped responses are passed + // through without being decompressed. + sitemapClient http.Client } type Config struct { @@ -116,6 +123,16 @@ func serve(cctx *cli.Context) error { }, }, }, + sitemapClient: http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ForceAttemptHTTP2: true, + DisableCompression: true, + }, + }, } // Create the HTTP server. @@ -337,6 +354,10 @@ func serve(cctx *cli.Context) error { // ipcc e.GET("/ipcc", server.WebIpCC) + // sitemap handlers + e.GET("/sitemap/users.xml.gz", server.handleSitemapUsersIndex) + e.GET("/sitemap/users/*", server.handleSitemapUsersSubpage) + if linkHost != "" { linkUrl, err := url.Parse(linkHost) if err != nil { @@ -753,3 +774,54 @@ func (srv *Server) WebIpCC(c echo.Context) error { } return c.JSON(200, outResponse) } + +func (srv *Server) handleSitemapUsersIndex(c echo.Context) error { + url := fmt.Sprintf("%s/external/sitemap/users.xml.gz", srv.cfg.appviewHost) + return srv.serveSitemapRequest(c, url, "user index") +} + +func (srv *Server) handleSitemapUsersSubpage(c echo.Context) error { + path := c.Param("*") + url := fmt.Sprintf("%s/external/sitemap/users/%s", srv.cfg.appviewHost, path) + return srv.serveSitemapRequest(c, url, "user subpage") +} + +func (srv *Server) serveSitemapRequest(c echo.Context, url, sitemapType string) error { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + slog.Error("failed to construct sitemap request", "err", err, "type", sitemapType) + return c.String(http.StatusInternalServerError, "Internal Server Error") + } + + resp, err := srv.sitemapClient.Do(req) + if err != nil { + slog.Error("failed to send sitemap request to appview", "err", err, "type", sitemapType) + return c.String(http.StatusInternalServerError, "Internal Server Error") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + buf, err := io.ReadAll(resp.Body) + if err != nil { + slog.Error("failed to read sitemap error response body", "err", err) + } + + slog.Error("invalid sitemap response code", + "err", err, + "type", sitemapType, + "code", resp.StatusCode, + "body", string(buf), + ) + return c.String(http.StatusInternalServerError, "Internal Server Error") + } + + c.Response().Header().Set("Content-Type", "application/xml") + c.Response().Header().Set("Content-Encoding", "gzip") + c.Response().WriteHeader(resp.StatusCode) + + if _, err = io.Copy(c.Response().Writer, resp.Body); err != nil { + slog.Error("failed to copy sitemap response body to client", "err", err, "type", sitemapType) + } + + return nil +} diff --git a/bskyweb/static/robots.txt b/bskyweb/static/robots.txt index 4f8510d18d..95950a64d9 100644 --- a/bskyweb/static/robots.txt +++ b/bskyweb/static/robots.txt @@ -7,3 +7,5 @@ # be ok. User-Agent: * Allow: / + +Sitemap: https://bsky.app/sitemap/users.xml.gz