# /robots.txt file example file
# most of it commented out

#User-agent: webcrawler
#Disallow:

#User-agent: lycra
#Disallow: /

User-agent: *              # Applies to all robots
Disallow: /css           # For internal use

User-agent: *              # Applies to all robots
Disallow: /db           # For internal use

User-agent: *              # Applies to all robots
Disallow: /headline           # For internal use

User-agent: *              # Applies to all robots
Disallow: /jq           # For internal use

User-agent: *              # Applies to all robots
Disallow: /ol           # For internal use

User-agent: *              # Applies to all robots
Disallow: /sj           # For internal use

User-agent: *              # Applies to all robots
Disallow: /stat           # For internal use