# app/views/robots/allow.text.erb # Note the .text extension

# ALLOW CERTAIN ROBOTS
# Baiduspider
User-agent: Baiduspider
Disallow: /
Disallow: /$
Disallow: /users/new
Disallow: /website/index
Disallow: /hotels/info

# others
User-Agent: *
Disallow: /
Allow: /$
Allow: /users/new
Allow: /website/index
Allow: /hotels/info