Kept the prototype out of search engines in the Website service.

Services/Website/Tests/App/AppTests.swift

@@ -205,6 +205,77 @@ struct AppTests {
         }
     }
 
+    @Test(arguments: [
+        "/", "/search?q=nightward", "/listing/nightward-1-cgc-9-8", "/shop/kestrel-row-comics", "/series/nightward",
+        "/grading-guide", "/events", "/sign-in", "/css/shared.css", "/robots.txt", "/sitemap.xml", "/health",
+        "/no-such-page", "/listing/missing", "/messages", "/search/",
+    ])
+    func `every response to be kept out of search engines`(
+        uri: String
+    ) async throws {
+        try await app(
+            staticFilesPath: staticFilesPath
+        ).test(.router) { client in
+            try await client.execute(
+                uri: uri,
+                method: .get
+            ) { response in
+                // Pages, files, the not-found page, a redirect to the sign-in, a trailing-slash redirect: all of them.
+                #expect(response.headers[.robotsTag]?.contains("noindex") == true)
+            }
+        }
+    }
+
+    @Test
+    func `the no-index mark to come off once the site is indexable`() async throws {
+        try await app(
+            staticFilesPath: staticFilesPath,
+            isIndexable: true
+        ).test(.router) { client in
+            try await client.execute(
+                uri: "/",
+                method: .get
+            ) { response in
+                #expect(response.headers[.robotsTag] == nil)
+            }
+
+            // The health checks mark themselves, whatever the site does.
+            try await client.execute(
+                uri: "/health",
+                method: .get
+            ) { response in
+                #expect(response.headers[.robotsTag] == "noindex")
+            }
+        }
+    }
+
+    @Test
+    func `crawlers to be let in, to read the no-index mark, and offered no sitemap`() async throws {
+        try await app(
+            staticFilesPath: staticFilesPath
+        ).test(.router) { client in
+            try await client.execute(
+                uri: "/robots.txt",
+                method: .get
+            ) { response in
+                let lines = String(buffer: response.body)
+                    .split(separator: "\n")
+                    .filter { !$0.hasPrefix("#") }
+
+                #expect(lines.contains("Disallow:"))
+                #expect(!lines.contains { $0.hasPrefix("Disallow: /") })
+                #expect(!lines.contains { $0.hasPrefix("Sitemap:") })
+            }
+
+            try await client.execute(
+                uri: "/sitemap.xml",
+                method: .get
+            ) { response in
+                #expect(!String(buffer: response.body).contains("<loc>"))
+            }
+        }
+    }
+
     @Test
     func `error page to be served for an unknown listing`() async throws {
         try await app(
@@ -610,38 +681,39 @@ private extension AppTests {
     
     func app(
         staticFilesPath: String,
-        strictTransportSecurity: String? = nil
+        strictTransportSecurity: String? = nil,
+        isIndexable: Bool? = nil
     ) async throws -> some ApplicationProtocol {
         try await application(
             reader: reader(
                 staticFilesPath: staticFilesPath,
-                strictTransportSecurity: strictTransportSecurity
+                strictTransportSecurity: strictTransportSecurity,
+                isIndexable: isIndexable
             )
         )
     }
-    
+
     func reader(
         staticFilesPath: String,
-        strictTransportSecurity: String? = nil
+        strictTransportSecurity: String? = nil,
+        isIndexable: Bool? = nil
     ) -> ConfigReader {
-        ConfigReader(providers: [{
-            if let strictTransportSecurity {
-                InMemoryProvider(values: [
-                    .HTTP.host: "127.0.0.1",
-                    .HTTP.port: "0",
-                    .Log.level: "trace",
-                    .Path.staticFiles: .init(stringLiteral: staticFilesPath),
-                    .Security.strictTransportSecurity: .init(stringLiteral: strictTransportSecurity)
-                ])
-            } else {
-                InMemoryProvider(values: [
-                    .HTTP.host: "127.0.0.1",
-                    .HTTP.port: "0",
-                    .Log.level: "trace",
-                    .Path.staticFiles: .init(stringLiteral: staticFilesPath),
-                ])
-            }
-        }()])
+        var values: [AbsoluteConfigKey: ConfigValue] = [
+            .HTTP.host: "127.0.0.1",
+            .HTTP.port: "0",
+            .Log.level: "trace",
+            .Path.staticFiles: .init(stringLiteral: staticFilesPath),
+        ]
+
+        if let strictTransportSecurity {
+            values[.Security.strictTransportSecurity] = .init(stringLiteral: strictTransportSecurity)
+        }
+
+        if let isIndexable {
+            values[.Site.indexable] = .init(booleanLiteral: isIndexable)
+        }
+
+        return ConfigReader(providers: [InMemoryProvider(values: values)])
     }
 
 }

Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift

@@ -0,0 +1,43 @@
+import Hummingbird
+import HummingbirdTesting
+import Infrastructure
+import Testing
+
+@testable import WebsiteLibrary
+
+@Suite(
+    "NoIndexMiddleware middleware",
+    .tags(.type)
+)
+struct NoIndexMiddlewareTests {
+
+    // MARK: Functional tests
+
+    @Test
+    func `marks what is served, what redirects, and what fails`() async throws {
+        let router = Router(context: WebsiteRequestContext.self)
+
+        router.add(middleware: NoIndexMiddleware())
+        router.get("/page") { _, _ in "A page" }
+        router.get("/moved") { _, _ in Response.redirect(to: "/page") }
+        router.get("/forbidden") { _, _ -> String in throw HTTPError(.forbidden) }
+        router.get("/marked") { _, _ in
+            Response(status: .ok, headers: [.robotsTag: "noindex"])
+        }
+
+        try await Application(router: router).test(.router) { client in
+            for (uri, status) in [("/page", HTTPResponse.Status.ok), ("/moved", .seeOther), ("/forbidden", .forbidden), ("/missing", .notFound)] {
+                try await client.execute(uri: uri, method: .get) { response in
+                    #expect(response.status == status)
+                    #expect(response.headers[.robotsTag] == "noindex, nofollow")
+                }
+            }
+
+            // One mark, not two: a response that came marked is marked over.
+            try await client.execute(uri: "/marked", method: .get) { response in
+                #expect(response.headers[values: .robotsTag] == ["noindex, nofollow"])
+            }
+        }
+    }
+
+}