diff --git a/static/robots.txt b/static/robots.txt index 2baf5c96..0b38a92f 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,3 +1,7 @@ +# https://platform.openai.com/docs/gptbot +User-agent: GPTBot +Disallow: / + User-agent: * Disallow: /search/ Disallow: /page/