bskyweb: add robots disallow and 'canonical' config flags (#4760)

bskyweb/cmd/bskyweb/main.go

@@ -94,6 +94,20 @@ func run(args []string) {
 					Value:    "",
 					EnvVars:  []string{"STATIC_CDN_HOST"},
 				},
+				&cli.BoolFlag{
+					Name:     "bsky-canonical-instance",
+					Usage:    "Enable if this is the canonical deployment (bsky.app)",
+					Value:    false,
+					Required: false,
+					EnvVars:  []string{"BSKY_CANONICAL_INSTANCE"},
+				},
+				&cli.BoolFlag{
+					Name:     "robots-disallow-all",
+					Usage:    "Crawling is allowed by default. Enable this flag to Disallow all",
+					Value:    false,
+					Required: false,
+					EnvVars:  []string{"ROBOTS_DISALLOW_ALL"},
+				},
 			},
 		},
 	}

bskyweb/cmd/bskyweb/server.go

@@ -63,6 +63,8 @@ func serve(cctx *cli.Context) error {
 	corsOrigins := cctx.StringSlice("cors-allowed-origins")
 	staticCDNHost := cctx.String("static-cdn-host")
 	staticCDNHost = strings.TrimSuffix(staticCDNHost, "/")
+	canonicalInstance := cctx.Bool("bsky-canonical-instance")
+	robotsDisallowAll := cctx.Bool("robots-disallow-all")
 
 	// Echo
 	e := echo.New()
@@ -204,13 +206,23 @@ func serve(cctx *cli.Context) error {
 		return http.FS(fsys)
 	}())
 
-	e.GET("/robots.txt", echo.WrapHandler(staticHandler))
+	// enable some special endpoints for the "canonical" deployment (bsky.app). not having these enabled should *not* impact regular operation
-	e.GET("/ips-v4", echo.WrapHandler(staticHandler))
+	if canonicalInstance {
-	e.GET("/ips-v6", echo.WrapHandler(staticHandler))
+		e.GET("/ips-v4", echo.WrapHandler(staticHandler))
-	e.GET("/.well-known/*", echo.WrapHandler(staticHandler))
+		e.GET("/ips-v6", echo.WrapHandler(staticHandler))
-	e.GET("/security.txt", func(c echo.Context) error {
+		e.GET("/security.txt", func(c echo.Context) error {
-		return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt")
+			return c.Redirect(http.StatusMovedPermanently, "/.well-known/security.txt")
-	})
+		})
+		e.GET("/.well-known/*", echo.WrapHandler(staticHandler))
+	}
+
+	// default to permissive, but Disallow all if flag set
+	if robotsDisallowAll {
+		e.File("/robots.txt", "static/robots-disallow-all.txt")
+	} else {
+		e.GET("/robots.txt", echo.WrapHandler(staticHandler))
+	}
+
 	e.GET("/iframe/youtube.html", echo.WrapHandler(staticHandler))
 	e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler)), func(next echo.HandlerFunc) echo.HandlerFunc {
 		return func(c echo.Context) error {

bskyweb/static/robots-disallow-all.txt

@@ -0,0 +1,3 @@
+# This is an development or self-hosted instance of the bsky web app, and crawling has been disallowed by the operator team.
+User-Agent: *
+Disallow: /