From 326299fc355b0a5781000f7451b5dd320cd1dfac Mon Sep 17 00:00:00 2001 From: Javier Cicchelli Date: Mon, 21 Sep 2026 17:36:39 +0200 Subject: [PATCH] Kept the prototype out of search engines in the Website service. --- Services/Website/.env.local | 3 + Services/Website/README.md | 3 + Services/Website/Resources/Static/robots.txt | 5 +- Services/Website/Resources/Static/sitemap.xml | 4 +- .../Sources/App/Extensions/App+Build.swift | 8 ++ .../Extensions/ConfigReader+Properties.swift | 10 ++ .../AbsoluteConfigKey+Constants.swift | 5 + .../Extensions/ConfigKey+Constants.swift | 2 + .../Middlewares/NoIndexMiddleware.swift | 43 +++++++ Services/Website/Tests/App/AppTests.swift | 116 ++++++++++++++---- .../Middlewares/NoIndexMiddlewareTests.swift | 43 +++++++ Services/Website/docker-compose.yml | 34 ++--- 12 files changed, 232 insertions(+), 44 deletions(-) create mode 100644 Services/Website/Sources/Library/Public/Middlewares/NoIndexMiddleware.swift create mode 100644 Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift diff --git a/Services/Website/.env.local b/Services/Website/.env.local index bd958b6..8319977 100644 --- a/Services/Website/.env.local +++ b/Services/Website/.env.local @@ -51,6 +51,9 @@ ANALYTICS_DOMAINS= # Whether the pages also embed the session recorder script: true | false ANALYTICS_RECORDER=false +# Whether search engines may index the site. Off by default: every response is marked `X-Robots-Tag: noindex, nofollow`. +# SITE_INDEXABLE=false + # --- Passkeys ---------------------------------------------------------------- # Origin the sign-in screen is served at; browsers bind passkeys to it. diff --git a/Services/Website/README.md b/Services/Website/README.md index 156f80d..d1843d4 100644 --- a/Services/Website/README.md +++ b/Services/Website/README.md @@ -42,6 +42,7 @@ The persistence backend runs as a `Fluent` service inside the application's Serv Requests pass through the middleware chain in this order (outermost first), then reach the routes: ``` +NoIndexMiddleware (X-Robots-Tag: noindex on every response, unless site.indexable) LogRequestsMiddleware → SecurityHeadersMiddleware (security headers on every response) → HTTPSRedirectMiddleware (301 to site.origin when forwarded over plain HTTP) @@ -235,6 +236,7 @@ every `GET`/`HEAD` whose path ends in a slash is answered with a `301` to the fo ### Site | Config key | Environment variable | Default | Description | | --- | --- | --- | --- | +| `site.indexable` | `SITE_INDEXABLE` | `false` | Whether search engines may index the site. While `false`, `NoIndexMiddleware` marks every response — pages, files, redirects, errors — `X-Robots-Tag: noindex, nofollow`. `robots.txt` stays open on purpose: a crawler barred from a page never reads its `noindex`, and may still list the bare URL. Before turning it on, restore the `Sitemap:` line and the sitemap's entries. | | `site.origin` | `SITE_ORIGIN` | _(set by bootstrap)_ | The public origin the site is served at (scheme and host, no trailing slash). The HTTPS redirect points at it, and the pages' `rel="canonical"` links and `hreflang` alternates derive from it. | Bootstrap writes the canonical URL it prompts for here, leaving it empty for the placeholder. An empty or non-HTTPS origin disables the [HTTPS redirect](#https-redirect), which `https.trustForwardedProto` must enable besides — a `301` is cached for a long time, so it is never issued at a host nobody named. An empty origin also leaves the pages without canonical URLs and language alternates, rather than building either against an empty host. @@ -479,6 +481,7 @@ The Makefile and Compose files read these from `.env` (or the environment). Prov | `HTTP_SERVER_NAME` | Runtime server name (default `CCNWebsite`). | | `SECURITY_STRICT_TRANSPORT_SECURITY` | HSTS header value (default `max-age=31536000; includeSubDomains`). | | `SITE_ORIGIN` | The public origin: scheme and host, no trailing slash. The local override points it at `http://localhost`. | +| `SITE_INDEXABLE` | `false` by default, which keeps the whole site out of search engines. | | `PASSKEYS_ORIGIN` | The origin browsers bind passkeys to: exactly where the sign-in screen is served, over HTTPS. Follows `SITE_ORIGIN` unless set apart. Left at the app's default (`http://localhost:8080`) every ceremony fails; changed later, it strands every passkey already registered. | | `ANALYTICS_WEBSITE_ID`, `ANALYTICS_DOMAINS`, `ANALYTICS_RECORDER` | Passed through to the tracker; an empty identifier leaves it off. | | `DATABASE_DRIVER` | `inMemory` or `postgres`. The production Compose file defaults it to `postgres`; the local override defaults back to the in-memory backend. | diff --git a/Services/Website/Resources/Static/robots.txt b/Services/Website/Resources/Static/robots.txt index 0515b0e..739ea67 100644 --- a/Services/Website/Resources/Static/robots.txt +++ b/Services/Website/Resources/Static/robots.txt @@ -1,7 +1,6 @@ # https://www.robotstxt.org/ -# Allow crawling of all content +# Crawling stays allowed on purpose. While the site is a prototype, every response carries `X-Robots-Tag: noindex, nofollow`, +# and a crawler barred here would never read it — it could still list the bare URLs it finds linked elsewhere. User-agent: * Disallow: - -Sitemap: https://ccn.rock-n-co.de/sitemap.xml diff --git a/Services/Website/Resources/Static/sitemap.xml b/Services/Website/Resources/Static/sitemap.xml index ede5fe4..72238ca 100644 --- a/Services/Website/Resources/Static/sitemap.xml +++ b/Services/Website/Resources/Static/sitemap.xml @@ -1,6 +1,4 @@ + - - https://ccn.rock-n-co.de - diff --git a/Services/Website/Sources/App/Extensions/App+Build.swift b/Services/Website/Sources/App/Extensions/App+Build.swift index 63ada33..14f00c3 100644 --- a/Services/Website/Sources/App/Extensions/App+Build.swift +++ b/Services/Website/Sources/App/Extensions/App+Build.swift @@ -64,6 +64,7 @@ func application( cacheControl: reader.cacheControl, compressionMinResponseSize: reader.compressionMinResponseSize, httpsRedirect: reader.httpsRedirect, + isIndexable: reader.isIndexable, rateLimit: reader.rateLimit, securityHeaders: reader.securityHeaders, // An unset origin leaves the pages without canonical URLs and language alternates, rather than @@ -181,6 +182,7 @@ private func logger( /// - analytics: the analytics tracker both pages embed, or `nil` to omit it. /// - cacheControl: the cache-control directives applied to the served static files. /// - compressionMinResponseSize: the minimum response body size, in bytes, before compression is applied. +/// - isIndexable: whether search engines may index the site; while `false`, every response is marked `noindex`. /// - httpsRedirect: the origin plain-HTTP requests are redirected to, and whether the forwarded-protocol header is trusted. /// - rateLimit: the rate limit configuration, currently applied to no route. /// - securityHeaders: the security headers applied to every response. @@ -201,6 +203,7 @@ private func router( cacheControl: CacheControl, compressionMinResponseSize: Int, httpsRedirect: HTTPSRedirectMiddleware.Configuration, + isIndexable: Bool, rateLimit: RateLimitMiddleware.Configuration, securityHeaders: SecurityHeadersMiddleware.Configuration, siteOrigin: String?, @@ -241,6 +244,11 @@ private func router( await home.refresh(with: .init(from: listings)) } + // Outermost, so the mark reaches redirects, errors, and served files alike. + if !isIndexable { + router.add(middleware: NoIndexMiddleware()) + } + router.addMiddleware { LogRequestsMiddleware(logLevel) SecurityHeadersMiddleware( diff --git a/Services/Website/Sources/App/Extensions/ConfigReader+Properties.swift b/Services/Website/Sources/App/Extensions/ConfigReader+Properties.swift index 3d80e30..e84376d 100644 --- a/Services/Website/Sources/App/Extensions/ConfigReader+Properties.swift +++ b/Services/Website/Sources/App/Extensions/ConfigReader+Properties.swift @@ -264,6 +264,16 @@ package extension ConfigReader { ) } + /// Whether search engines may index the site, read from the `site.indexable` key. + /// + /// Off by default: the prototype holds demo data, and a sign-in anyone can use. While off, every response is marked `noindex`. + var isIndexable: Bool { + bool( + forKey: .Site.indexable, + default: false + ) + } + /// The origin passkeys are bound to, read from the `passkeys.origin` key. var passkeysOrigin: String { string( diff --git a/Services/Website/Sources/Library/Public/Extensions/AbsoluteConfigKey+Constants.swift b/Services/Website/Sources/Library/Public/Extensions/AbsoluteConfigKey+Constants.swift index 216af97..9226406 100644 --- a/Services/Website/Sources/Library/Public/Extensions/AbsoluteConfigKey+Constants.swift +++ b/Services/Website/Sources/Library/Public/Extensions/AbsoluteConfigKey+Constants.swift @@ -82,6 +82,11 @@ extension AbsoluteConfigKey { /// The absolute configuration key for the directory the static files are served from. public static let staticFiles: AbsoluteConfigKey = .init(.Path.staticFiles) } + /// A namespace for the site configuration keys, as absolute keys. + public enum Site { + /// The absolute configuration key for letting search engines index the site. + public static let indexable: AbsoluteConfigKey = .init(.Site.indexable) + } /// A namespace for the security headers configuration keys, as absolute keys. public enum Security { /// The absolute configuration key for the `Content-Security-Policy` header value. diff --git a/Services/Website/Sources/Library/Public/Extensions/ConfigKey+Constants.swift b/Services/Website/Sources/Library/Public/Extensions/ConfigKey+Constants.swift index edec7dd..98fb455 100644 --- a/Services/Website/Sources/Library/Public/Extensions/ConfigKey+Constants.swift +++ b/Services/Website/Sources/Library/Public/Extensions/ConfigKey+Constants.swift @@ -107,5 +107,7 @@ extension ConfigKey { public enum Site { /// The configuration key for the public origin the site is served at (scheme and host, no trailing slash). public static let origin: ConfigKey = "site.origin" + /// The configuration key for letting search engines index the site. + public static let indexable: ConfigKey = "site.indexable" } } diff --git a/Services/Website/Sources/Library/Public/Middlewares/NoIndexMiddleware.swift b/Services/Website/Sources/Library/Public/Middlewares/NoIndexMiddleware.swift new file mode 100644 index 0000000..99ca79b --- /dev/null +++ b/Services/Website/Sources/Library/Public/Middlewares/NoIndexMiddleware.swift @@ -0,0 +1,43 @@ +import Hummingbird +import Infrastructure + +/// Keeps the site out of search engines: every response — pages, files, redirects, errors — is marked `X-Robots-Tag: noindex, nofollow`. +/// +/// A header rather than `robots.txt`, which stays open on purpose: a crawler barred from a page never reads its `noindex`, and may still +/// list the bare URL from links elsewhere. A header also covers what carries no markup, such as the photos sellers upload. +public struct NoIndexMiddleware: RouterMiddleware { + + // MARK: Constants + + /// The directive stamped on every response. + public static var directive: String { "noindex, nofollow" } + + // MARK: Initializers + + /// Creates the middleware. + public init() {} + + // MARK: Methods + + public func handle( + _ request: Request, + context: Context, + next: (Request, Context) async throws -> Response + ) async throws -> Response { + do { + var response = try await next(request, context) + + response.headers[.robotsTag] = Self.directive + + return response + } catch let error as HTTPError { + // An error thrown past this point becomes a response further out: the mark travels with it. + var error = error + + error.headers[.robotsTag] = Self.directive + + throw error + } + } + +} diff --git a/Services/Website/Tests/App/AppTests.swift b/Services/Website/Tests/App/AppTests.swift index 72a0978..c5f3f41 100644 --- a/Services/Website/Tests/App/AppTests.swift +++ b/Services/Website/Tests/App/AppTests.swift @@ -205,6 +205,77 @@ struct AppTests { } } + @Test(arguments: [ + "/", "/search?q=nightward", "/listing/nightward-1-cgc-9-8", "/shop/kestrel-row-comics", "/series/nightward", + "/grading-guide", "/events", "/sign-in", "/css/shared.css", "/robots.txt", "/sitemap.xml", "/health", + "/no-such-page", "/listing/missing", "/messages", "/search/", + ]) + func `every response to be kept out of search engines`( + uri: String + ) async throws { + try await app( + staticFilesPath: staticFilesPath + ).test(.router) { client in + try await client.execute( + uri: uri, + method: .get + ) { response in + // Pages, files, the not-found page, a redirect to the sign-in, a trailing-slash redirect: all of them. + #expect(response.headers[.robotsTag]?.contains("noindex") == true) + } + } + } + + @Test + func `the no-index mark to come off once the site is indexable`() async throws { + try await app( + staticFilesPath: staticFilesPath, + isIndexable: true + ).test(.router) { client in + try await client.execute( + uri: "/", + method: .get + ) { response in + #expect(response.headers[.robotsTag] == nil) + } + + // The health checks mark themselves, whatever the site does. + try await client.execute( + uri: "/health", + method: .get + ) { response in + #expect(response.headers[.robotsTag] == "noindex") + } + } + } + + @Test + func `crawlers to be let in, to read the no-index mark, and offered no sitemap`() async throws { + try await app( + staticFilesPath: staticFilesPath + ).test(.router) { client in + try await client.execute( + uri: "/robots.txt", + method: .get + ) { response in + let lines = String(buffer: response.body) + .split(separator: "\n") + .filter { !$0.hasPrefix("#") } + + #expect(lines.contains("Disallow:")) + #expect(!lines.contains { $0.hasPrefix("Disallow: /") }) + #expect(!lines.contains { $0.hasPrefix("Sitemap:") }) + } + + try await client.execute( + uri: "/sitemap.xml", + method: .get + ) { response in + #expect(!String(buffer: response.body).contains("")) + } + } + } + @Test func `error page to be served for an unknown listing`() async throws { try await app( @@ -610,38 +681,39 @@ private extension AppTests { func app( staticFilesPath: String, - strictTransportSecurity: String? = nil + strictTransportSecurity: String? = nil, + isIndexable: Bool? = nil ) async throws -> some ApplicationProtocol { try await application( reader: reader( staticFilesPath: staticFilesPath, - strictTransportSecurity: strictTransportSecurity + strictTransportSecurity: strictTransportSecurity, + isIndexable: isIndexable ) ) } - + func reader( staticFilesPath: String, - strictTransportSecurity: String? = nil + strictTransportSecurity: String? = nil, + isIndexable: Bool? = nil ) -> ConfigReader { - ConfigReader(providers: [{ - if let strictTransportSecurity { - InMemoryProvider(values: [ - .HTTP.host: "127.0.0.1", - .HTTP.port: "0", - .Log.level: "trace", - .Path.staticFiles: .init(stringLiteral: staticFilesPath), - .Security.strictTransportSecurity: .init(stringLiteral: strictTransportSecurity) - ]) - } else { - InMemoryProvider(values: [ - .HTTP.host: "127.0.0.1", - .HTTP.port: "0", - .Log.level: "trace", - .Path.staticFiles: .init(stringLiteral: staticFilesPath), - ]) - } - }()]) + var values: [AbsoluteConfigKey: ConfigValue] = [ + .HTTP.host: "127.0.0.1", + .HTTP.port: "0", + .Log.level: "trace", + .Path.staticFiles: .init(stringLiteral: staticFilesPath), + ] + + if let strictTransportSecurity { + values[.Security.strictTransportSecurity] = .init(stringLiteral: strictTransportSecurity) + } + + if let isIndexable { + values[.Site.indexable] = .init(booleanLiteral: isIndexable) + } + + return ConfigReader(providers: [InMemoryProvider(values: values)]) } } diff --git a/Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift b/Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift new file mode 100644 index 0000000..4dab8af --- /dev/null +++ b/Services/Website/Tests/Library/Cases/Public/Middlewares/NoIndexMiddlewareTests.swift @@ -0,0 +1,43 @@ +import Hummingbird +import HummingbirdTesting +import Infrastructure +import Testing + +@testable import WebsiteLibrary + +@Suite( + "NoIndexMiddleware middleware", + .tags(.type) +) +struct NoIndexMiddlewareTests { + + // MARK: Functional tests + + @Test + func `marks what is served, what redirects, and what fails`() async throws { + let router = Router(context: WebsiteRequestContext.self) + + router.add(middleware: NoIndexMiddleware()) + router.get("/page") { _, _ in "A page" } + router.get("/moved") { _, _ in Response.redirect(to: "/page") } + router.get("/forbidden") { _, _ -> String in throw HTTPError(.forbidden) } + router.get("/marked") { _, _ in + Response(status: .ok, headers: [.robotsTag: "noindex"]) + } + + try await Application(router: router).test(.router) { client in + for (uri, status) in [("/page", HTTPResponse.Status.ok), ("/moved", .seeOther), ("/forbidden", .forbidden), ("/missing", .notFound)] { + try await client.execute(uri: uri, method: .get) { response in + #expect(response.status == status) + #expect(response.headers[.robotsTag] == "noindex, nofollow") + } + } + + // One mark, not two: a response that came marked is marked over. + try await client.execute(uri: "/marked", method: .get) { response in + #expect(response.headers[values: .robotsTag] == ["noindex, nofollow"]) + } + } + } + +} diff --git a/Services/Website/docker-compose.yml b/Services/Website/docker-compose.yml index 5b9447b..ed66d7a 100644 --- a/Services/Website/docker-compose.yml +++ b/Services/Website/docker-compose.yml @@ -20,33 +20,35 @@ services: ports: - "${HOST_PORT:-8080}:8080" environment: + # Empty leaves the tracker off. + ANALYTICS_WEBSITE_ID: ${ANALYTICS_WEBSITE_ID:-} + ANALYTICS_DOMAINS: ${ANALYTICS_DOMAINS:-} + ANALYTICS_RECORDER: ${ANALYTICS_RECORDER:-false} + DATABASE_DRIVER: ${DATABASE_DRIVER:-postgres} + DATABASE_HOST: ${DATABASE_HOST:?DATABASE_HOST is required} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-ccn} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} + DATABASE_POOL_MAX_PER_EVENT_LOOP: ${DATABASE_POOL_MAX_PER_EVENT_LOOP:-4} + DATABASE_TLS: ${DATABASE_TLS:-require} + DATABASE_USERNAME: ${DATABASE_USERNAME:-ccn} LOG_LEVEL: ${LOG_LEVEL:-info} HTTP_SERVER_NAME: ${HTTP_SERVER_NAME:-CCNWebsite} - # The public origin: scheme and host, no trailing slash. The HTTPS redirect and the canonical links derive from it. - SITE_ORIGIN: ${SITE_ORIGIN:-https://ccn.rock-n-co.de} + HTTPS_TRUST_FORWARDED_PROTO: "${HTTPS_TRUST_FORWARDED_PROTO:-true}" # The origin browsers bind passkeys to: exactly where the sign-in screen is served — same scheme, same host, no # `www.` either way. Left at the app's default (http://localhost:8080) every ceremony fails, so it follows # SITE_ORIGIN unless set apart. It must be HTTPS: browsers run WebAuthn nowhere else but localhost, and it is # what marks the session cookie `Secure`. PASSKEYS_ORIGIN: ${PASSKEYS_ORIGIN:-${SITE_ORIGIN:-https://ccn.rock-n-co.de}} - # Empty leaves the tracker off. - ANALYTICS_WEBSITE_ID: ${ANALYTICS_WEBSITE_ID:-} - ANALYTICS_DOMAINS: ${ANALYTICS_DOMAINS:-} - ANALYTICS_RECORDER: ${ANALYTICS_RECORDER:-false} # Falls back to the policy the app ships with; set it in `.env` to allow the analytics origin, # which must match `String.Analytics.origin`. + RATE_LIMIT_TRUST_FORWARDED_FOR: "${RATE_LIMIT_TRUST_FORWARDED_FOR:-true}" SECURITY_CONTENT_SECURITY_POLICY: "${SECURITY_CONTENT_SECURITY_POLICY:-default-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'}" SECURITY_STRICT_TRANSPORT_SECURITY: "${SECURITY_STRICT_TRANSPORT_SECURITY:-max-age=31536000; includeSubDomains}" - HTTPS_TRUST_FORWARDED_PROTO: "${HTTPS_TRUST_FORWARDED_PROTO:-true}" - RATE_LIMIT_TRUST_FORWARDED_FOR: "${RATE_LIMIT_TRUST_FORWARDED_FOR:-true}" - DATABASE_DRIVER: ${DATABASE_DRIVER:-postgres} - DATABASE_HOST: ${DATABASE_HOST:?DATABASE_HOST is required} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME:-ccn} - DATABASE_USERNAME: ${DATABASE_USERNAME:-ccn} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:?DATABASE_PASSWORD is required} - DATABASE_TLS: ${DATABASE_TLS:-require} - DATABASE_POOL_MAX_PER_EVENT_LOOP: ${DATABASE_POOL_MAX_PER_EVENT_LOOP:-4} + # The public origin: scheme and host, no trailing slash. The HTTPS redirect and the canonical links derive from it. + SITE_ORIGIN: ${SITE_ORIGIN:-https://ccn.rock-n-co.de} + # Off while the site is a prototype: every response is marked `X-Robots-Tag: noindex, nofollow`. + SITE_INDEXABLE: "${SITE_INDEXABLE:-false}" volumes: # The photos sellers upload. The rest of the image is read-only, and without the volume a redeploy would lose them. - website-uploads:/app/Resources/Static/uploads