Search: limit number of sections and domain to 10K
ES 7.x has this number as default for nested objects.
It gives an error if we try to index more than that.
Closes
@@ -16,6 +16,9 @@ log = logging.getLogger(__name__)
class BaseParser:
+ # Limit that matches the ``index.mapping.nested_objects.limit`` ES setting.
+ max_inner_documents = 10000
+
def __init__(self, version):
self.version = version
self.project = self.version.project
@@ -140,6 +143,21 @@ class BaseParser:
except Exception as e:
log.info('Unable to index section: %s', str(e))
+ def _get_sections(self, title, body):
+ """Get the first `self.max_inner_documents` sections."""
+ iterator = self._parse_sections(title=title, body=body)
+ sections = list(itertools.islice(iterator, 0, self.max_inner_documents))
+ try:
+ next(iterator)
+ except StopIteration:
+ pass
+ else:
+ log.warning(
+ 'Limit of inner sections exceeded. project=%s version=%s limit=%d',
+ self.project.slug, self.version.slug, self.max_inner_documents,
+ )
+ return sections
+
def _clean_body(self, body):
"""
Removes nodes with irrelevant content before parsing its sections.
@@ -363,7 +381,7 @@ class SphinxParser(BaseParser):
if 'body' in data:
try:
body = HTMLParser(data['body'])
- sections = list(self._parse_sections(title=title, body=body.body))
+ sections = self._get_sections(title=title, body=body.body)
except Exception as e:
log.info('Unable to index sections for: %s', fjson_path)
@@ -423,10 +441,15 @@ class SphinxParser(BaseParser):
"domain-id-1": "docstrings for the domain-id-1",
"domain-id-2": "docstrings for the domain-id-2",
}
+
+ .. note::
+
+ Only the first `self.max_inner_documents` domains are returned.
"""
domain_data = {}
dl_tags = body.css('dl')
+ number_of_domains = 0
for dl_tag in dl_tags:
@@ -441,6 +464,13 @@ class SphinxParser(BaseParser):
if id_:
docstrings = self._parse_domain_tag(desc)
domain_data[id_] = docstrings
+ number_of_domains += 1
+ if number_of_domains >= self.max_inner_documents:
+ log.warning(
+ 'Limit of inner domains exceeded. project=%s version=%s limit=%i',
+ self.project.slug, self.version.slug, self.max_inner_documents,
+ )
+ break
except Exception:
log.exception('Error parsing docstring for domains')
@@ -497,7 +527,7 @@ class MkDocsParser(BaseParser):
sections = []
if body:
title = self._get_page_title(body, html) or page
- sections = list(self._parse_sections(title, body))
+ sections = self._get_sections(title=title, body=body)
else:
log.info(
'Page doesn\'t look like it has valid content, skipping. '
