acetheking987/bsky-social-app
1
0
Fork 0
Code Issues Actions Packages Projects Releases 1 Wiki Activity

bskyweb: meta noindex tag for non-indexable pages (#10803)

Browse Source
This commit is contained in:
Michael Black
2026-06-09 15:54:54 -05:00
committed by GitHub
parent db2e190f4c
commit 38fb9c2165
3 changed files with 133 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bskyweb/cmd/bskyweb/render_test.go
+54
View File
@@ -43,6 +43,13 @@ func extractJSONLD(t *testing.T, html string) string {
return strings.TrimSpace(m[1])
}
func TestRenderBase_NoindexMeta(t *testing.T) {
html := renderTemplate(t, "base.html", pongo2.Context{"noindex": true, "nofollow": true})
if !strings.Contains(html, `<meta name="robots" content="noindex, nofollow">`) {
t.Errorf("expected combined noindex,nofollow meta; got:\n%s", html)
}
}
func TestRenderPost_EmitsJSONLD(t *testing.T) {
pv := makePostView("alice.bsky.social", "did:plc:alice", "abc123", "hello")
ld, err := buildPostJSONLD(pv, nil, "https://bsky.app/profile/alice.bsky.social/post/abc123", hideEmbedLabels, hideReplyLabels)
@@ -243,3 +250,50 @@ func TestRenderPost_VideoWithoutThumbnailEmitsOGVideo(t *testing.T) {
t.Errorf("og:video:type should emit even without imgThumbUrls; got:\n%s", html)
}
}
// Auth-required posts must emit noindex,nofollow so the stub page (no body
// text, no comments) is not indexed.
func TestRenderPost_AuthRequiredNoindex(t *testing.T) {
html := renderTemplate(t, "post.html", pongo2.Context{
"requiresAuth": true,
"profileHandle": "alice.bsky.social",
"requestURI": "https://bsky.app/profile/alice.bsky.social/post/abc123",
"canonicalURL": "https://bsky.app/profile/alice.bsky.social/post/abc123",
"noindex": true,
"nofollow": true,
})
if !strings.Contains(html, `<meta name="robots" content="noindex, nofollow">`) {
t.Errorf("auth-required post should emit noindex,nofollow; got:\n%s", html)
}
}
// Auth-required profiles must emit noindex,nofollow.
func TestRenderProfile_AuthRequiredNoindex(t *testing.T) {
pv := newProfileViewDetailed()
html := renderTemplate(t, "profile.html", pongo2.Context{
"profileView": pv,
"requestURI": "https://bsky.app/profile/alice.bsky.social",
"requiresAuth": true,
"noindex": true,
"nofollow": true,
})
if !strings.Contains(html, `<meta name="robots" content="noindex, nofollow">`) {
t.Errorf("auth-required profile should emit noindex,nofollow; got:\n%s", html)
}
}
// Public posts must NOT emit a robots meta tag. Guards against an accidental
// flip of the noindex flag for indexable pages.
func TestRenderPost_PublicNoNoindex(t *testing.T) {
pv := makePostView("alice.bsky.social", "did:plc:alice", "abc123", "hello")
ld, _ := buildPostJSONLD(pv, nil, "https://bsky.app/profile/alice.bsky.social/post/abc123", hideEmbedLabels, hideReplyLabels)
html := renderTemplate(t, "post.html", pongo2.Context{
"postView": pv,
"requestURI": "https://bsky.app/profile/alice.bsky.social/post/abc123",
"canonicalURL": "https://bsky.app/profile/alice.bsky.social/post/abc123",
"postJSONLD": ld,
})
if strings.Contains(html, `<meta name="robots"`) {
t.Errorf("public post should not emit robots meta; got:\n%s", html)
}
}
bskyweb/cmd/bskyweb/server.go
+78 -43
View File
@@ -296,50 +296,50 @@ func serve(cctx *cli.Context) error {
// generic routes
e.GET("/hashtag/:tag", server.WebGeneric)
e.GET("/topic/:topic", server.WebGeneric)
e.GET("/search", server.WebGeneric)
e.GET("/feeds", server.WebGeneric)
e.GET("/notifications", server.WebGeneric)
e.GET("/notifications/settings", server.WebGeneric)
e.GET("/notifications/activity", server.WebGeneric)
e.GET("/lists", server.WebGeneric)
e.GET("/moderation", server.WebGeneric)
e.GET("/moderation/modlists", server.WebGeneric)
e.GET("/moderation/muted-accounts", server.WebGeneric)
e.GET("/moderation/blocked-accounts", server.WebGeneric)
e.GET("/moderation/verification-settings", server.WebGeneric)
e.GET("/settings", server.WebGeneric)
e.GET("/settings/language", server.WebGeneric)
e.GET("/settings/app-passwords", server.WebGeneric)
e.GET("/settings/following-feed", server.WebGeneric)
e.GET("/settings/saved-feeds", server.WebGeneric)
e.GET("/settings/threads", server.WebGeneric)
e.GET("/settings/external-embeds", server.WebGeneric)
e.GET("/settings/accessibility", server.WebGeneric)
e.GET("/settings/appearance", server.WebGeneric)
e.GET("/settings/account", server.WebGeneric)
e.GET("/settings/automation-label", server.WebGeneric)
e.GET("/settings/privacy-and-security", server.WebGeneric)
e.GET("/settings/privacy-and-security/activity", server.WebGeneric)
e.GET("/settings/content-and-media", server.WebGeneric)
e.GET("/settings/interests", server.WebGeneric)
e.GET("/settings/about", server.WebGeneric)
e.GET("/settings/notifications", server.WebGeneric)
e.GET("/sys/debug", server.WebGeneric)
e.GET("/sys/debug-mod", server.WebGeneric)
e.GET("/sys/log", server.WebGeneric)
e.GET("/search", server.WebGenericNoindex)
e.GET("/feeds", server.WebGenericNoindex)
e.GET("/notifications", server.WebGenericNoindex)
e.GET("/notifications/settings", server.WebGenericNoindex)
e.GET("/notifications/activity", server.WebGenericNoindex)
e.GET("/lists", server.WebGenericNoindex)
e.GET("/moderation", server.WebGenericNoindex)
e.GET("/moderation/modlists", server.WebGenericNoindex)
e.GET("/moderation/muted-accounts", server.WebGenericNoindex)
e.GET("/moderation/blocked-accounts", server.WebGenericNoindex)
e.GET("/moderation/verification-settings", server.WebGenericNoindex)
e.GET("/settings", server.WebGenericNoindex)
e.GET("/settings/language", server.WebGenericNoindex)
e.GET("/settings/app-passwords", server.WebGenericNoindex)
e.GET("/settings/following-feed", server.WebGenericNoindex)
e.GET("/settings/saved-feeds", server.WebGenericNoindex)
e.GET("/settings/threads", server.WebGenericNoindex)
e.GET("/settings/external-embeds", server.WebGenericNoindex)
e.GET("/settings/accessibility", server.WebGenericNoindex)
e.GET("/settings/appearance", server.WebGenericNoindex)
e.GET("/settings/account", server.WebGenericNoindex)
e.GET("/settings/automation-label", server.WebGenericNoindex)
e.GET("/settings/privacy-and-security", server.WebGenericNoindex)
e.GET("/settings/privacy-and-security/activity", server.WebGenericNoindex)
e.GET("/settings/content-and-media", server.WebGenericNoindex)
e.GET("/settings/interests", server.WebGenericNoindex)
e.GET("/settings/about", server.WebGenericNoindex)
e.GET("/settings/notifications", server.WebGenericNoindex)
e.GET("/sys/debug", server.WebGenericNoindex)
e.GET("/sys/debug-mod", server.WebGenericNoindex)
e.GET("/sys/log", server.WebGenericNoindex)
e.GET("/support", server.WebGeneric)
e.GET("/support/privacy", server.WebGeneric)
e.GET("/support/tos", server.WebGeneric)
e.GET("/support/community-guidelines", server.WebGeneric)
e.GET("/support/copyright", server.WebGeneric)
e.GET("/intent/compose", server.WebGeneric)
e.GET("/intent/verify-email", server.WebGeneric)
e.GET("/intent/age-assurance", server.WebGeneric)
e.GET("/messages", server.WebGeneric)
e.GET("/messages/inbox", server.WebGeneric)
e.GET("/messages/:conversation", server.WebGeneric)
e.GET("/messages/:conversation/settings", server.WebGeneric)
e.GET("/messages/:conversation/requests", server.WebGeneric)
e.GET("/intent/compose", server.WebGenericNoindexNofollow)
e.GET("/intent/verify-email", server.WebGenericNoindexNofollow)
e.GET("/intent/age-assurance", server.WebGenericNoindexNofollow)
e.GET("/messages", server.WebGenericNoindex)
e.GET("/messages/inbox", server.WebGenericNoindex)
e.GET("/messages/:conversation", server.WebGenericNoindex)
e.GET("/messages/:conversation/settings", server.WebGenericNoindex)
e.GET("/messages/:conversation/requests", server.WebGenericNoindex)
// profile endpoints; only first populates info
e.GET("/profile/:handleOrDID", server.WebProfile)
@@ -363,14 +363,14 @@ func serve(cctx *cli.Context) error {
// starter packs
e.GET("/starter-pack/:handleOrDID/:rkey", server.WebStarterPack)
e.GET("/starter-pack-short/:code", server.WebGeneric)
e.GET("/starter-pack-short/:code", server.WebGenericNoindex)
e.GET("/start/:handleOrDID/:rkey", server.WebStarterPack)
// chat invites
e.GET("/chat/:code", server.WebChatInvite)
// bookmarks
e.GET("/saved", server.WebGeneric)
e.GET("/saved", server.WebGenericNoindex)
// ipcc
e.GET("/ipcc", server.WebIpCC)
@@ -437,6 +437,8 @@ func (srv *Server) NewTemplateContext() pongo2.Context {
return pongo2.Context{
"staticCDNHost": srv.cfg.staticCDNHost,
"favicon": fmt.Sprintf("%s/static/favicon.png", srv.cfg.staticCDNHost),
"noindex": false,
"nofollow": false,
}
}
@@ -489,10 +491,38 @@ func (srv *Server) LinkProxyMiddleware(url *url.URL) echo.MiddlewareFunc {
)
}
// renderOptions controls per-request rendering flags for the generic web handler.
type renderOptions struct {
noindex bool
nofollow bool
}
// webGeneric returns a handler that renders the base SPA shell with the given
// render options applied to the template context.
func (srv *Server) webGeneric(c echo.Context, o renderOptions) error {
data := srv.NewTemplateContext()
data["noindex"] = o.noindex
data["nofollow"] = o.nofollow
return c.Render(http.StatusOK, "base.html", data)
}
// handler for endpoint that have no specific server-side handling
func (srv *Server) WebGeneric(c echo.Context) error {
data := srv.NewTemplateContext()
return c.Render(http.StatusOK, "base.html", data)
return srv.webGeneric(c, renderOptions{})
}
// handler for routes that should not be indexed by search engines
// (e.g. auth-only user-state surfaces, internal/debug pages, action/intent dispatch URLs, search results)
func (srv *Server) WebGenericNoindex(c echo.Context) error {
return srv.webGeneric(c, renderOptions{noindex: true})
}
// handler for action/intent dispatch URLs (e.g. /intent/compose). These accept
// arbitrary query parameters from arbitrary third-party referrers, so we treat
// them as link-graph dead-ends in addition to noindex. Anything legitimately
// reachable from a hydrated intent page is also reachable via its canonical URL.
func (srv *Server) WebGenericNoindexNofollow(c echo.Context) error {
return srv.webGeneric(c, renderOptions{noindex: true, nofollow: true})
}
func (srv *Server) WebHome(c echo.Context) error {
@@ -567,6 +597,8 @@ func (srv *Server) WebPost(c echo.Context) error {
data["canonicalURL"] = canonicalURL
}
data["requiresAuth"] = true
data["noindex"] = true
data["nofollow"] = true
data["profileHandle"] = pv.Handle
if pv.DisplayName != nil {
data["profileDisplayName"] = *pv.DisplayName
@@ -673,6 +705,7 @@ func (srv *Server) WebChatInvite(c echo.Context) error {
req := c.Request()
ctx := req.Context()
data := srv.NewTemplateContext()
data["noindex"] = true
data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path)
code := c.Param("code")
@@ -760,6 +793,8 @@ func (srv *Server) WebProfile(c echo.Context) error {
}
} else {
data["requiresAuth"] = true
data["noindex"] = true
data["nofollow"] = true
}
if jsonld, err := buildProfileJSONLD(pv, recentPosts, hideEmbedLabels, hideReplyLabels); err == nil {