bskyweb: meta noindex tag for non-indexable pages

bskyweb/cmd/bskyweb/server.go

@@ -296,50 +296,50 @@ func serve(cctx *cli.Context) error {
 	// generic routes
 	e.GET("/hashtag/:tag", server.WebGeneric)
 	e.GET("/topic/:topic", server.WebGeneric)
-	e.GET("/search", server.WebGeneric)
-	e.GET("/feeds", server.WebGeneric)
-	e.GET("/notifications", server.WebGeneric)
-	e.GET("/notifications/settings", server.WebGeneric)
-	e.GET("/notifications/activity", server.WebGeneric)
-	e.GET("/lists", server.WebGeneric)
-	e.GET("/moderation", server.WebGeneric)
-	e.GET("/moderation/modlists", server.WebGeneric)
-	e.GET("/moderation/muted-accounts", server.WebGeneric)
-	e.GET("/moderation/blocked-accounts", server.WebGeneric)
-	e.GET("/moderation/verification-settings", server.WebGeneric)
-	e.GET("/settings", server.WebGeneric)
-	e.GET("/settings/language", server.WebGeneric)
-	e.GET("/settings/app-passwords", server.WebGeneric)
-	e.GET("/settings/following-feed", server.WebGeneric)
-	e.GET("/settings/saved-feeds", server.WebGeneric)
-	e.GET("/settings/threads", server.WebGeneric)
-	e.GET("/settings/external-embeds", server.WebGeneric)
-	e.GET("/settings/accessibility", server.WebGeneric)
-	e.GET("/settings/appearance", server.WebGeneric)
-	e.GET("/settings/account", server.WebGeneric)
-	e.GET("/settings/automation-label", server.WebGeneric)
-	e.GET("/settings/privacy-and-security", server.WebGeneric)
-	e.GET("/settings/privacy-and-security/activity", server.WebGeneric)
-	e.GET("/settings/content-and-media", server.WebGeneric)
-	e.GET("/settings/interests", server.WebGeneric)
-	e.GET("/settings/about", server.WebGeneric)
-	e.GET("/settings/notifications", server.WebGeneric)
-	e.GET("/sys/debug", server.WebGeneric)
-	e.GET("/sys/debug-mod", server.WebGeneric)
-	e.GET("/sys/log", server.WebGeneric)
+	e.GET("/search", server.WebGenericNoindex)
+	e.GET("/feeds", server.WebGenericNoindex)
+	e.GET("/notifications", server.WebGenericNoindex)
+	e.GET("/notifications/settings", server.WebGenericNoindex)
+	e.GET("/notifications/activity", server.WebGenericNoindex)
+	e.GET("/lists", server.WebGenericNoindex)
+	e.GET("/moderation", server.WebGenericNoindex)
+	e.GET("/moderation/modlists", server.WebGenericNoindex)
+	e.GET("/moderation/muted-accounts", server.WebGenericNoindex)
+	e.GET("/moderation/blocked-accounts", server.WebGenericNoindex)
+	e.GET("/moderation/verification-settings", server.WebGenericNoindex)
+	e.GET("/settings", server.WebGenericNoindex)
+	e.GET("/settings/language", server.WebGenericNoindex)
+	e.GET("/settings/app-passwords", server.WebGenericNoindex)
+	e.GET("/settings/following-feed", server.WebGenericNoindex)
+	e.GET("/settings/saved-feeds", server.WebGenericNoindex)
+	e.GET("/settings/threads", server.WebGenericNoindex)
+	e.GET("/settings/external-embeds", server.WebGenericNoindex)
+	e.GET("/settings/accessibility", server.WebGenericNoindex)
+	e.GET("/settings/appearance", server.WebGenericNoindex)
+	e.GET("/settings/account", server.WebGenericNoindex)
+	e.GET("/settings/automation-label", server.WebGenericNoindex)
+	e.GET("/settings/privacy-and-security", server.WebGenericNoindex)
+	e.GET("/settings/privacy-and-security/activity", server.WebGenericNoindex)
+	e.GET("/settings/content-and-media", server.WebGenericNoindex)
+	e.GET("/settings/interests", server.WebGenericNoindex)
+	e.GET("/settings/about", server.WebGenericNoindex)
+	e.GET("/settings/notifications", server.WebGenericNoindex)
+	e.GET("/sys/debug", server.WebGenericNoindex)
+	e.GET("/sys/debug-mod", server.WebGenericNoindex)
+	e.GET("/sys/log", server.WebGenericNoindex)
 	e.GET("/support", server.WebGeneric)
 	e.GET("/support/privacy", server.WebGeneric)
 	e.GET("/support/tos", server.WebGeneric)
 	e.GET("/support/community-guidelines", server.WebGeneric)
 	e.GET("/support/copyright", server.WebGeneric)
-	e.GET("/intent/compose", server.WebGeneric)
-	e.GET("/intent/verify-email", server.WebGeneric)
-	e.GET("/intent/age-assurance", server.WebGeneric)
-	e.GET("/messages", server.WebGeneric)
-	e.GET("/messages/inbox", server.WebGeneric)
-	e.GET("/messages/:conversation", server.WebGeneric)
-	e.GET("/messages/:conversation/settings", server.WebGeneric)
-	e.GET("/messages/:conversation/requests", server.WebGeneric)
+	e.GET("/intent/compose", server.WebGenericNoindexNofollow)
+	e.GET("/intent/verify-email", server.WebGenericNoindexNofollow)
+	e.GET("/intent/age-assurance", server.WebGenericNoindexNofollow)
+	e.GET("/messages", server.WebGenericNoindex)
+	e.GET("/messages/inbox", server.WebGenericNoindex)
+	e.GET("/messages/:conversation", server.WebGenericNoindex)
+	e.GET("/messages/:conversation/settings", server.WebGenericNoindex)
+	e.GET("/messages/:conversation/requests", server.WebGenericNoindex)
 
 	// profile endpoints; only first populates info
 	e.GET("/profile/:handleOrDID", server.WebProfile)
@@ -363,14 +363,14 @@ func serve(cctx *cli.Context) error {
 
 	// starter packs
 	e.GET("/starter-pack/:handleOrDID/:rkey", server.WebStarterPack)
-	e.GET("/starter-pack-short/:code", server.WebGeneric)
+	e.GET("/starter-pack-short/:code", server.WebGenericNoindex)
 	e.GET("/start/:handleOrDID/:rkey", server.WebStarterPack)
 
 	// chat invites
 	e.GET("/chat/:code", server.WebChatInvite)
 
 	// bookmarks
-	e.GET("/saved", server.WebGeneric)
+	e.GET("/saved", server.WebGenericNoindex)
 
 	// ipcc
 	e.GET("/ipcc", server.WebIpCC)
@@ -437,6 +437,8 @@ func (srv *Server) NewTemplateContext() pongo2.Context {
 	return pongo2.Context{
 		"staticCDNHost": srv.cfg.staticCDNHost,
 		"favicon":       fmt.Sprintf("%s/static/favicon.png", srv.cfg.staticCDNHost),
+		"noindex":       false,
+		"nofollow":      false,
 	}
 }
 
@@ -489,10 +491,40 @@ func (srv *Server) LinkProxyMiddleware(url *url.URL) echo.MiddlewareFunc {
 	)
 }
 
+// renderOptions controls per-request rendering flags for the generic web handler.
+type renderOptions struct {
+	noindex  bool
+	nofollow bool
+}
+
+// webGeneric returns a handler that renders the base SPA shell with the given
+// render options applied to the template context.
+func (srv *Server) webGeneric(o renderOptions) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		data := srv.NewTemplateContext()
+		data["noindex"] = o.noindex
+		data["nofollow"] = o.nofollow
+		return c.Render(http.StatusOK, "base.html", data)
+	}
+}
+
 // handler for endpoint that have no specific server-side handling
 func (srv *Server) WebGeneric(c echo.Context) error {
-	data := srv.NewTemplateContext()
-	return c.Render(http.StatusOK, "base.html", data)
+	return srv.webGeneric(renderOptions{})(c)
+}
+
+// handler for routes that should not be indexed by search engines
+// (e.g. auth-only user-state surfaces, internal/debug pages, action/intent dispatch URLs, search results)
+func (srv *Server) WebGenericNoindex(c echo.Context) error {
+	return srv.webGeneric(renderOptions{noindex: true})(c)
+}
+
+// handler for action/intent dispatch URLs (e.g. /intent/compose). These accept
+// arbitrary query parameters from arbitrary third-party referrers, so we treat
+// them as link-graph dead-ends in addition to noindex. Anything legitimately
+// reachable from a hydrated intent page is also reachable via its canonical URL.
+func (srv *Server) WebGenericNoindexNofollow(c echo.Context) error {
+	return srv.webGeneric(renderOptions{noindex: true, nofollow: true})(c)
 }
 
 func (srv *Server) WebHome(c echo.Context) error {
@@ -673,6 +705,7 @@ func (srv *Server) WebChatInvite(c echo.Context) error {
 	req := c.Request()
 	ctx := req.Context()
 	data := srv.NewTemplateContext()
+	data["noindex"] = true
 	data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path)
 
 	code := c.Param("code")

bskyweb/templates/base.html

@@ -139,6 +139,7 @@
   <meta name="theme-color">
   <meta name="application-name" content="Bluesky">
   <meta name="generator" content="bskyweb">
+  {% if noindex %}<meta name="robots" content="noindex{% if nofollow %}, nofollow{% endif %}">{% endif %}
   <meta property="og:site_name" content="Bluesky Social">
   <meta property="og:logo" content="{{ favicon }}">
   <meta name="twitter:site" content="@bluesky" />