Do not serve the robots.txt file if private or version not active/built
@@ -233,12 +233,23 @@ def robots_txt(request, project):
If the user added a ``robots.txt`` in the "default version" of the project,
we serve it directly.
"""
- if project.privacy_level == constants.PRIVATE:
- # If project is private, we disallow the whole site
- raise HttpResponse('User-agent: *\nDisallow: /\n')
-
# Use the ``robots.txt`` file from the default version configured
version_slug = project.get_default_version()
+ version = project.versions.get(slug=version_slug)
+
+ no_serve_robots_txt = any(
+ # If project is private or,
+ project.privacy_level == constants.PRIVATE,
+ # default version is private or,
+ version.privacy_level == constants.PRIVATE,
+ # default version is not active or,
+ not version.active,
+ # default version is not built
+ not version.built,
+ )
+ if no_serve_robots_txt:
+ # ... we do return a 404
+ raise Http404()
filename = resolve_path(
project,
