added robots.txt

This commit is contained in:
Arian Furrer 2024-04-28 13:09:30 +02:00
parent e3ae991614
commit 8d0acdc86a
2 changed files with 19 additions and 0 deletions

View File

@ -19,6 +19,8 @@ module.exports = function(eleventyConfig) {
eleventyConfig.addPassthroughCopy("_src/style.css");
// add _media folder
eleventyConfig.addPassthroughCopy("_src/_media");
// add robots.txt
eleventyConfig.addPassthroughCopy({ 'src/robots.txt': '/robots.txt' });
// shortcodes
eleventyConfig.addShortcode("year", () => {

17
_src/robots.txt Normal file
View File

@ -0,0 +1,17 @@
#SOURCE: <https://www.didiermary.fr/bloquer-ai-bots-chatgpt-openai/>
# AI crawlers
User-agent: Amazonbot 🆕
User-agent: anthropic-ai
User-agent: Applebot 🆕
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: cohere-ai
User-agent: Diffbot 🆕
User-Agent: FacebookBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: Omgili1
User-agent: PerplexityBot 🆕
User-agent: YouBot 🆕
Disallow: /