Kept the prototype out of search engines in the Website service.

Services/Website/.env.local

@@ -51,6 +51,9 @@ ANALYTICS_DOMAINS=
 # Whether the pages also embed the session recorder script: true | false
 ANALYTICS_RECORDER=false
 
+# Whether search engines may index the site. Off by default: every response is marked `X-Robots-Tag: noindex, nofollow`.
+# SITE_INDEXABLE=false
+
 # --- Passkeys ----------------------------------------------------------------
 
 # Origin the sign-in screen is served at; browsers bind passkeys to it.

Services/Website/README.md

@@ -42,6 +42,7 @@ The persistence backend runs as a `Fluent` service inside the application's Serv
 
 Requests pass through the middleware chain in this order (outermost first), then reach the routes:
 ```
+NoIndexMiddleware                        (X-Robots-Tag: noindex on every response, unless site.indexable)
 LogRequestsMiddleware
   → SecurityHeadersMiddleware            (security headers on every response)
     → HTTPSRedirectMiddleware            (301 to site.origin when forwarded over plain HTTP)
@@ -235,6 +236,7 @@ every `GET`/`HEAD` whose path ends in a slash is answered with a `301` to the fo
 ### Site
 | Config key | Environment variable | Default | Description |
 | --- | --- | --- | --- |
+| `site.indexable` | `SITE_INDEXABLE` | `false` | Whether search engines may index the site. While `false`, `NoIndexMiddleware` marks every response — pages, files, redirects, errors — `X-Robots-Tag: noindex, nofollow`. `robots.txt` stays open on purpose: a crawler barred from a page never reads its `noindex`, and may still list the bare URL. Before turning it on, restore the `Sitemap:` line and the sitemap's entries. |
 | `site.origin` | `SITE_ORIGIN` | _(set by bootstrap)_ | The public origin the site is served at (scheme and host, no trailing slash). The HTTPS redirect points at it, and the pages' `rel="canonical"` links and `hreflang` alternates derive from it. |
 
 Bootstrap writes the canonical URL it prompts for here, leaving it empty for the placeholder. An empty or non-HTTPS origin disables the [HTTPS redirect](#https-redirect), which `https.trustForwardedProto` must enable besides — a `301` is cached for a long time, so it is never issued at a host nobody named. An empty origin also leaves the pages without canonical URLs and language alternates, rather than building either against an empty host.
@@ -479,6 +481,7 @@ The Makefile and Compose files read these from `.env` (or the environment). Prov
 | `HTTP_SERVER_NAME` | Runtime server name (default `CCNWebsite`). |
 | `SECURITY_STRICT_TRANSPORT_SECURITY` | HSTS header value (default `max-age=31536000; includeSubDomains`). |
 | `SITE_ORIGIN` | The public origin: scheme and host, no trailing slash. The local override points it at `http://localhost`. |
+| `SITE_INDEXABLE` | `false` by default, which keeps the whole site out of search engines. |
 | `PASSKEYS_ORIGIN` | The origin browsers bind passkeys to: exactly where the sign-in screen is served, over HTTPS. Follows `SITE_ORIGIN` unless set apart. Left at the app's default (`http://localhost:8080`) every ceremony fails; changed later, it strands every passkey already registered. |
 | `ANALYTICS_WEBSITE_ID`, `ANALYTICS_DOMAINS`, `ANALYTICS_RECORDER` | Passed through to the tracker; an empty identifier leaves it off. |
 | `DATABASE_DRIVER` | `inMemory` or `postgres`. The production Compose file defaults it to `postgres`; the local override defaults back to the in-memory backend. |

Services/Website/Resources/Static/robots.txt

@@ -1,7 +1,6 @@
 # https://www.robotstxt.org/
 
-# Allow crawling of all content
+# Crawling stays allowed on purpose. While the site is a prototype, every response carries `X-Robots-Tag: noindex, nofollow`,
+# and a crawler barred here would never read it — it could still list the bare URLs it finds linked elsewhere.
 User-agent: *
 Disallow:
-
-Sitemap: https://ccn.rock-n-co.de/sitemap.xml

Services/Website/Resources/Static/sitemap.xml

@@ -1,6 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!-- Empty while the site is a prototype kept out of search engines: nothing here is meant to be indexed. -->
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <url>
-    <loc>https://ccn.rock-n-co.de</loc>
-  </url>
 </urlset>

Services/Website/Sources/App/Extensions/App+Build.swift

@@ -64,6 +64,7 @@ func application(
             cacheControl: reader.cacheControl,
             compressionMinResponseSize: reader.compressionMinResponseSize,
             httpsRedirect: reader.httpsRedirect,
+            isIndexable: reader.isIndexable,
             rateLimit: reader.rateLimit,
             securityHeaders: reader.securityHeaders,
             // An unset origin leaves the pages without canonical URLs and language alternates, rather than
@@ -181,6 +182,7 @@ private func logger(
 ///   - analytics: the analytics tracker both pages embed, or `nil` to omit it.
 ///   - cacheControl: the cache-control directives applied to the served static files.
 ///   - compressionMinResponseSize: the minimum response body size, in bytes, before compression is applied.
+///   - isIndexable: whether search engines may index the site; while `false`, every response is marked `noindex`.
 ///   - httpsRedirect: the origin plain-HTTP requests are redirected to, and whether the forwarded-protocol header is trusted.
 ///   - rateLimit: the rate limit configuration, currently applied to no route.
 ///   - securityHeaders: the security headers applied to every response.
@@ -201,6 +203,7 @@ private func router(
     cacheControl: CacheControl,
     compressionMinResponseSize: Int,
     httpsRedirect: HTTPSRedirectMiddleware<AppRequestContext>.Configuration,
+    isIndexable: Bool,
     rateLimit: RateLimitMiddleware<AppRequestContext>.Configuration,
     securityHeaders: SecurityHeadersMiddleware<AppRequestContext>.Configuration,
     siteOrigin: String?,
@@ -241,6 +244,11 @@ private func router(
         await home.refresh(with: .init(from: listings))
     }
 
+    // Outermost, so the mark reaches redirects, errors, and served files alike.
+    if !isIndexable {
+        router.add(middleware: NoIndexMiddleware())
+    }
+
     router.addMiddleware {
         LogRequestsMiddleware(logLevel)
         SecurityHeadersMiddleware(

Services/Website/Sources/App/Extensions/ConfigReader+Properties.swift

@@ -264,6 +264,16 @@ package extension ConfigReader {
         )
     }
 
+    /// Whether search engines may index the site, read from the `site.indexable` key.
+    ///
+    /// Off by default: the prototype holds demo data, and a sign-in anyone can use. While off, every response is marked `noindex`.
+    var isIndexable: Bool {
+        bool(
+            forKey: .Site.indexable,
+            default: false
+        )
+    }
+
     /// The origin passkeys are bound to, read from the `passkeys.origin` key.
     var passkeysOrigin: String {
         string(

Services/Website/Sources/Library/Public/Extensions/AbsoluteConfigKey+Constants.swift

@@ -82,6 +82,11 @@ extension AbsoluteConfigKey {
         /// The absolute configuration key for the directory the static files are served from.
         public static let staticFiles: AbsoluteConfigKey = .init(.Path.staticFiles)
     }
+    /// A namespace for the site configuration keys, as absolute keys.
+    public enum Site {
+        /// The absolute configuration key for letting search engines index the site.
+        public static let indexable: AbsoluteConfigKey = .init(.Site.indexable)
+    }
     /// A namespace for the security headers configuration keys, as absolute keys.
     public enum Security {
         /// The absolute configuration key for the `Content-Security-Policy` header value.

Services/Website/Sources/Library/Public/Extensions/ConfigKey+Constants.swift

@@ -107,5 +107,7 @@ extension ConfigKey {
     public enum Site {
         /// The configuration key for the public origin the site is served at (scheme and host, no trailing slash).
         public static let origin: ConfigKey = "site.origin"
+        /// The configuration key for letting search engines index the site.
+        public static let indexable: ConfigKey = "site.indexable"
     }
 }

Services/Website/Sources/Library/Public/Middlewares/NoIndexMiddleware.swift

@@ -0,0 +1,43 @@
+import Hummingbird
+import Infrastructure
+
+/// Keeps the site out of search engines: every response — pages, files, redirects, errors — is marked `X-Robots-Tag: noindex, nofollow`.
+///
+/// A header rather than `robots.txt`, which stays open on purpose: a crawler barred from a page never reads its `noindex`, and may still
+/// list the bare URL from links elsewhere. A header also covers what carries no markup, such as the photos sellers upload.
+public struct NoIndexMiddleware<Context: RequestContext>: RouterMiddleware {
+
+    // MARK: Constants
+
+    /// The directive stamped on every response.
+    public static var directive: String { "noindex, nofollow" }
+
+    // MARK: Initializers
+
+    /// Creates the middleware.
+    public init() {}
+
+    // MARK: Methods
+
+    public func handle(
+        _ request: Request,
+        context: Context,
+        next: (Request, Context) async throws -> Response
+    ) async throws -> Response {
+        do {
+            var response = try await next(request, context)
+
+            response.headers[.robotsTag] = Self.directive
+
+            return response
+        } catch let error as HTTPError {
+            // An error thrown past this point becomes a response further out: the mark travels with it.
+            var error = error
+
+            error.headers[.robotsTag] = Self.directive
+
+            throw error
+        }
+    }
+
+}

Services/Website/Tests/App/AppTests.swift

@@ -205,6 +205,77 @@ struct AppTests {
         }
     }
 
+    @Test(arguments: [
+        "/", "/search?q=nightward", "/listing/nightward-1-cgc-9-8", "/shop/kestrel-row-comics", "/series/nightward",
+        "/grading-guide", "/events", "/sign-in", "/css/shared.css", "/robots.txt", "/sitemap.xml", "/health",
+        "/no-such-page", "/listing/missing", "/messages", "/search/",
+    ])
+    func `every response to be kept out of search engines`(
+        uri: String
+    ) async throws {
+        try await app(
+            staticFilesPath: staticFilesPath
+        ).test(.router) { client in
+            try await client.execute(
+                uri: uri,
+                method: .get
+            ) { response in
+                // Pages, files, the not-found page, a redirect to the sign-in, a trailing-slash redirect: all of them.
+                #expect(response.headers[.robotsTag]?.contains("noindex") == true)
+            }
+        }
+    }
+
+    @Test
+    func `the no-index mark to come off once the site is indexable`() async throws {
+        try await app(
+            staticFilesPath: staticFilesPath,
+            isIndexable: true
+        ).test(.router) { client in
+            try await client.execute(
+                uri: "/",
+                method: .get
+            ) { response in
+                #expect(response.headers[.robotsTag] == nil)
+            }
+
+            // The health checks mark themselves, whatever the site does.
+            try await client.execute(
+                uri: "/health",
+                method: .get
+            ) { response in
+                #expect(response.headers[.robotsTag] == "noindex")
+            }
+        }
+    }
+
+    @Test
+    func `crawlers to be let in, to read the no-index mark, and offered no sitemap`() async throws {
+        try await app(
+            staticFilesPath: staticFilesPath
+        ).test(.router) { client in
+            try await client.execute(
+                uri: "/robots.txt",
+                method: .get
+            ) { response in
+                let lines = String(buffer: response.body)
+                    .split(separator: "\n")
+                    .filter { !$0.hasPrefix("#") }
+
+                #expect(lines.contains("Disallow:"))
+                #expect(!lines.contains { $0.hasPrefix("Disallow: /") })
+                #expect(!lines.contains { $0.hasPrefix("Sitemap:") })
+            }
+
+            try await client.execute(
+                uri: "/sitemap.xml",
+                method: .get
+            ) { response in
+                #expect(!String(buffer: response.body).contains("<loc>"))
+            }
+        }
+    }
+
     @Test
     func `error page to be served for an unknown listing`() async throws {
         try await app(
@@ -610,38 +681,39 @@ private extension AppTests {
     
     func app(
         staticFilesPath: String,
-        strictTransportSecurity: String? = nil
+        strictTransportSecurity: String? = nil,
+        isIndexable: Bool? = nil
     ) async throws -> some ApplicationProtocol {
         try await application(
             reader: reader(
                 staticFilesPath: staticFilesPath,
-                strictTransportSecurity: strictTransportSecurity
+                strictTransportSecurity: strictTransportSecurity,
+                isIndexable: isIndexable
             )
         )
     }
 
     func reader(
         staticFilesPath: String,
-        strictTransportSecurity: String? = nil
+        strictTransportSecurity: String? = nil,
+        isIndexable: Bool? = nil
     ) -> ConfigReader {
-        ConfigReader(providers: [{
+        var values: [AbsoluteConfigKey: ConfigValue] = [
-            if let strictTransportSecurity {
+            .HTTP.host: "127.0.0.1",
-                InMemoryProvider(values: [
+            .HTTP.port: "0",
-                    .HTTP.host: "127.0.0.1",
+            .Log.level: "trace",
-                    .HTTP.port: "0",
+            .Path.staticFiles: .init(stringLiteral: staticFilesPath),
-                    .Log.level: "trace",
+        ]
-                    .Path.staticFiles: .init(stringLiteral: staticFilesPath),
+
-                    .Security.strictTransportSecurity: .init(stringLiteral: strictTransportSecurity)
+        if let strictTransportSecurity {
-                ])
+            values[.Security.strictTransportSecurity] = .init(stringLiteral: strictTransportSecurity)
-            } else {
+        }
-                InMemoryProvider(values: [
+
-                    .HTTP.host: "127.0.0.1",
+        if let isIndexable {
-                    .HTTP.port: "0",
+            values[.Site.indexable] = .init(booleanLiteral: isIndexable)
-                    .Log.level: "trace",
+        }
-                    .Path.staticFiles: .init(stringLiteral: staticFilesPath),
+
-                ])
+        return ConfigReader(providers: [InMemoryProvider(values: values)])
-            }
-        }()])
     }
 
 }

Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift

@@ -0,0 +1,43 @@
+import Hummingbird
+import HummingbirdTesting
+import Infrastructure
+import Testing
+
+@testable import WebsiteLibrary
+
+@Suite(
+    "NoIndexMiddleware middleware",
+    .tags(.type)
+)
+struct NoIndexMiddlewareTests {
+
+    // MARK: Functional tests
+
+    @Test
+    func `marks what is served, what redirects, and what fails`() async throws {
+        let router = Router(context: WebsiteRequestContext.self)
+
+        router.add(middleware: NoIndexMiddleware())
+        router.get("/page") { _, _ in "A page" }
+        router.get("/moved") { _, _ in Response.redirect(to: "/page") }
+        router.get("/forbidden") { _, _ -> String in throw HTTPError(.forbidden) }
+        router.get("/marked") { _, _ in
+            Response(status: .ok, headers: [.robotsTag: "noindex"])
+        }
+
+        try await Application(router: router).test(.router) { client in
+            for (uri, status) in [("/page", HTTPResponse.Status.ok), ("/moved", .seeOther), ("/forbidden", .forbidden), ("/missing", .notFound)] {
+                try await client.execute(uri: uri, method: .get) { response in
+                    #expect(response.status == status)
+                    #expect(response.headers[.robotsTag] == "noindex, nofollow")
+                }
+            }
+
+            // One mark, not two: a response that came marked is marked over.
+            try await client.execute(uri: "/marked", method: .get) { response in
+                #expect(response.headers[values: .robotsTag] == ["noindex, nofollow"])
+            }
+        }
+    }
+
+}

Services/Website/docker-compose.yml

@@ -20,33 +20,35 @@ services:
     ports:
       - "${HOST_PORT:-8080}:8080"
     environment:
+      # Empty leaves the tracker off.
+      ANALYTICS_WEBSITE_ID: ${ANALYTICS_WEBSITE_ID:-}
+      ANALYTICS_DOMAINS: ${ANALYTICS_DOMAINS:-}
+      ANALYTICS_RECORDER: ${ANALYTICS_RECORDER:-false}
+      DATABASE_DRIVER: ${DATABASE_DRIVER:-postgres}
+      DATABASE_HOST: ${DATABASE_HOST:?DATABASE_HOST is required}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+      DATABASE_NAME: ${DATABASE_NAME:-ccn}
+      DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required}
+      DATABASE_POOL_MAX_PER_EVENT_LOOP: ${DATABASE_POOL_MAX_PER_EVENT_LOOP:-4}
+      DATABASE_TLS: ${DATABASE_TLS:-require}
+      DATABASE_USERNAME: ${DATABASE_USERNAME:-ccn}
       LOG_LEVEL: ${LOG_LEVEL:-info}
       HTTP_SERVER_NAME: ${HTTP_SERVER_NAME:-CCNWebsite}
-      # The public origin: scheme and host, no trailing slash. The HTTPS redirect and the canonical links derive from it.
+      HTTPS_TRUST_FORWARDED_PROTO: "${HTTPS_TRUST_FORWARDED_PROTO:-true}"
-      SITE_ORIGIN: ${SITE_ORIGIN:-https://ccn.rock-n-co.de}
       # The origin browsers bind passkeys to: exactly where the sign-in screen is served — same scheme, same host, no
       # `www.` either way. Left at the app's default (http://localhost:8080) every ceremony fails, so it follows
       # SITE_ORIGIN unless set apart. It must be HTTPS: browsers run WebAuthn nowhere else but localhost, and it is
       # what marks the session cookie `Secure`.
       PASSKEYS_ORIGIN: ${PASSKEYS_ORIGIN:-${SITE_ORIGIN:-https://ccn.rock-n-co.de}}
-      # Empty leaves the tracker off.
-      ANALYTICS_WEBSITE_ID: ${ANALYTICS_WEBSITE_ID:-}
-      ANALYTICS_DOMAINS: ${ANALYTICS_DOMAINS:-}
-      ANALYTICS_RECORDER: ${ANALYTICS_RECORDER:-false}
       # Falls back to the policy the app ships with; set it in `.env` to allow the analytics origin,
       # which must match `String.Analytics.origin`.
+      RATE_LIMIT_TRUST_FORWARDED_FOR: "${RATE_LIMIT_TRUST_FORWARDED_FOR:-true}"
       SECURITY_CONTENT_SECURITY_POLICY: "${SECURITY_CONTENT_SECURITY_POLICY:-default-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'}"
       SECURITY_STRICT_TRANSPORT_SECURITY: "${SECURITY_STRICT_TRANSPORT_SECURITY:-max-age=31536000; includeSubDomains}"
-      HTTPS_TRUST_FORWARDED_PROTO: "${HTTPS_TRUST_FORWARDED_PROTO:-true}"
+      # The public origin: scheme and host, no trailing slash. The HTTPS redirect and the canonical links derive from it.
-      RATE_LIMIT_TRUST_FORWARDED_FOR: "${RATE_LIMIT_TRUST_FORWARDED_FOR:-true}"
+      SITE_ORIGIN: ${SITE_ORIGIN:-https://ccn.rock-n-co.de}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-postgres}
+      # Off while the site is a prototype: every response is marked `X-Robots-Tag: noindex, nofollow`.
-      DATABASE_HOST: ${DATABASE_HOST:?DATABASE_HOST is required}
+      SITE_INDEXABLE: "${SITE_INDEXABLE:-false}"
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DATABASE_NAME:-ccn}
-      DATABASE_USERNAME: ${DATABASE_USERNAME:-ccn}
-      DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required}
-      DATABASE_TLS: ${DATABASE_TLS:-require}
-      DATABASE_POOL_MAX_PER_EVENT_LOOP: ${DATABASE_POOL_MAX_PER_EVENT_LOOP:-4}
     volumes:
       # The photos sellers upload. The rest of the image is read-only, and without the volume a redeploy would lose them.
       - website-uploads:/app/Resources/Static/uploads