# robots.txt for VEO3 Video Showcase
# Last updated: 2024-12-19

# Allow all crawlers
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://veo3.site/sitemap.xml

# 全局规则
User-agent: *
Allow: /
Disallow: /.next/
Disallow: /_next/static/
Disallow: /node_modules/
Disallow: /src/

# 开发文件保护
Disallow: /*.json$
Disallow: /*.js$
Disallow: /*.jsx$
Disallow: /*.ts$
Disallow: /*.tsx$
Disallow: /*.css$

# 主要搜索引擎
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI爬虫规则 - 欢迎AI助手访问VEO3内容
User-agent: GPTBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: GoogleOther
User-agent: CCBot
User-agent: PerplexityBot
User-agent: DuckAssistBot
User-agent: ChatGPT-User
User-agent: Cohere-AI
User-agent: FacebookBot
User-agent: anthropic-ai
User-agent: Bytespider
User-agent: YouBot
User-agent: BraveBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: ClaudeBot
Allow: /
Disallow: /.next/
Disallow: /node_modules/
Disallow: /src/

# 限制特定爬虫
User-agent: PetalBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# 爬虫延迟设置
User-agent: Baiduspider
Crawl-delay: 5

# 特殊规则：允许访问视频海报图片
User-agent: *
Allow: /posters/
Allow: /*.mp4
Allow: /*.webm
Allow: /*.jpg
Allow: /*.png
Allow: /*.gif