# codescan.kr robots.txt
# 주의: robots 규칙상 봇은 '자기 UA와 가장 정확히 일치하는 그룹 하나만' 따른다.
# 이름을 지목한 그룹에 Allow: / 만 쓰면 그 봇에게는 아래 * 그룹의 Disallow가
# 전혀 적용되지 않는다 — 2026-09-05까지 Yeti(네이버)·Bingbot·AI 답변봇이
# 관리자 경로와 고객 스캔 결과(/security/result/)까지 크롤링 가능한 상태였다
# (결과 페이지는 X-Robots-Tag: noindex라 색인은 안 됐지만 크롤 자체가 열려 있었다).
# → 이름 지목 그룹마다 공통 Disallow를 반드시 함께 적는다.

User-agent: Googlebot
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: Yeti
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: Bingbot
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

# AI 답변 생성 엔진 — 브랜드 노출 허용
User-agent: PerplexityBot
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: SearchGPT
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

# AI 답변 엔진 — 브랜드 답변 인용 노출 허용 (GEO 정책)
User-agent: ChatGPT-User
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: OAI-SearchBot
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: ClaudeBot
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

User-agent: Claude-Web
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/

# AI 학습 크롤러 차단 (모델 학습 데이터 제공 거부)
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Amazonbot
Disallow: /

# FacebookBot은 Meta 검색·추천 인덱싱용 — Threads 채널 운영과 상충해 허용.
# 학습 데이터 수집은 Meta-ExternalAgent가 담당하므로 그쪽만 차단 유지.
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: *
Allow: /
Disallow: /lyh0202/
Disallow: /security/dashboard/
Disallow: /security/api/
Disallow: /security/scan/sse/
Disallow: /security/scan/email-fallback/
Disallow: /security/scan/recent/
Disallow: /security/mypage/
Disallow: /security/result/
Disallow: /continuous/
Disallow: /media/

# LLM-friendly guide: https://codescan.kr/llms.txt
Sitemap: https://codescan.kr/sitemap.xml