import os
import csv
import functools

AMPERSAND = '&'
LESS_THAN = '<'
MORE_THAN = '>'
APOSTROFE = '’'
ASTERISCO = '•'

ROLES = ['HQ', 'Troops', 'Elites', 'Fast Attack', 'Heavy Support', 'Flyer', 'Dedicated Transport', 'No Force Org Slot']

ANGELS_OF_DEATH = 'And They Shall Know No Fear, Bolter Discipline, Combat Doctrines'

KEY_WORDS_IN_RULES = ['When', 'Re-roll', 'During', 'If', 're-roll', 'Ballistic',
                      'target', 'suffer', 'to hit', 'Each', 'select', 'Instead', 'shooting', 'For the purposes',
                      'Each time', 'gain', 'controls', 'Roll', 'firing', 'You can', 'This model', 'cannot', 'must',
                      'subtract', 'rolls', 'The first time', 'roll', 'In addition', 'Models', ' If', 'can be included',
                      'preventing', 'After', 'At the start', 'each', 'Command phase', 'if', 'added', 'Can', 'Cannot',
                      'counts', 'number of dice', 'characteristic', 'model']

RULE_BOOKS = ['Codex:', 'Psychic Awakening']

MODELS_STAT = ['M', 'WS', 'BS', 'S', 'T', 'W', 'A', 'Ld', 'Save'], ['Inv', 'FNP']
WEAPONS_STATS = ['Range', 'Type', 'S', 'AP', 'D', 'Abilities']
PSY_POW_STATS = ['Psychic Power', 'Warp Charge', 'Range', 'Details']
PSY_MODEL_ABILITY_STATS = ['Cast', 'Deny', 'Powers Known', 'Other']

class ParseFromHtml:
    extension = '.txt'
    book_references_list = ['Psychic Awakening']
    serial_No = 10000
    path = os.getcwd()
    address = './ParsedData'
    own_dir = '/ready_data_'
    full_path = f'{address}{own_dir}'

    def __init__(self, battle_scribe_html_file):

        # Army data:
        self.name = None
        self.pts = None
        self.CP = 0
        self.battle_type = None
        self.detachment = None
        self.faction = None
        self.subfaction = None
        self.codex = None

        self.amount_of_units = None

        self.roles = []
        self.units = []
        self.psychic_powers = []
        self.army_rules = {}  # all of them
        self.warlord = None

        # METHODS TO CONSTRUCT OBJECT, this data is only useful for program insides
        self.file = battle_scribe_html_file
        self.raw_data_text = []  # all the html with out code in txt
        self.general_rules_text = []
        self.army_rules_names = []
        self.army_rules_list = []
        self.general_rules_text = list(zip(self.army_rules_names, self.army_rules_list))
        for rule in self.general_rules_text:
            self.army_rules[rule[0]] = rule[1]
        self.all_unit_chunks = []

        # system
        self.start_system()

        # all parse data organise in:
        self.create_general_dir()
        self.create_army_dir()

        # empty memory
        self.general_rules_text = []
        self.army_rules_names = []
        self.raw_units_text = []
        self.raw_psy_text = []

        #  todo: -weapon, stats ( + record, csv, sqlite)

    def __repr__(self):
        if not self.psychic_powers:
            self.psychic_powers = None
        return f"""
Army Name: {self.name}
Army Points: {self.pts}
Army Detachment: {self.detachment}
Army Faction: {self.faction}
Army CPs: {self.CP}
Army Battle Type: {self.battle_type}
Army Sub-Faction: {self.subfaction}
Army Rules: {self.army_rules}
Army Roles: {self.roles}
Army Amount of Units: {self.amount_of_units}
Army Psychic Powers: {self.psychic_powers}
Army Units: {self.units}"""

    def start_system(self):
        self.open_file_to_data()
        self.cut_first_part()  # up to HQ lines
        self.cut_last_part()
        self.cut_mid_size_tags_lines()
        self.mark_start_of_unit()
        self.deal_with_small_tags()

        self.erase_Categories_Configuration_lines()
        self.erase_Detachment_Command_Cost_lines()
        self.erase_configuration_CP_line()
        self.erase_blank_lines()
        self.emplace_more_les_than_signs()
        self.emplace_ampersand()
        self.emplace_apostrofe()
        self.emplace_asterisco()

        self.populate_general_rules_data()  # separates general rules and units

        self.get_army_name()
        self.get_army_pts()
        self.get_detachment()
        self.get_faction()
        self.get_CP()
        self.get_battle_type()
        self.get_first_line_rule()
        self.get_sub_faction()
        self.get_faction_ability()

        self.create_general_dir()
        self.create_army_dir()
        self.record_raw_data_in_txt()

        self.erase_pages_and_useless_data()
        self.fix_army_rules_names()
        self.get_the_rest_of_the_general_rules()
        self.make_general_rules_text_pair_odd_lines()
        self.get_the_damned_rules()
        self.record_rules_in_csv()
        # end of rules block

        self.get_psychic_powers()
        self.parse_psy_powers()
        self.record_psypow_in_csv()
        # end of psy pow block

        self.clean_unit_pages_references()
        self.how_many_each_roles_in_army()

        self.test()

    def create_general_dir(self):
        if not os.path.exists(f'{self.address}'):
            os.mkdir(f'{self.address}')

    def create_army_dir(self):
        address = f'{self.full_path}{self.name}'
        if not os.path.exists(address):
            os.mkdir(address)
        self.address = address

    # PREPARE RAW DATA BlOCK
    def open_file_to_data(self):
        with open(self.file, 'r') as file:
            data = file.readlines()

        for line in data:
            if line != '\n':
                self.raw_data_text.append(line.strip())

    def record_raw_data_in_txt(self):
        """inner method: prepare the html in txt format to be parsed"""

        filename = self.name.strip()
        name = filename + self.extension
        if not os.path.exists(f'{self.address}'):
            self.address = f'{self.address}/plain_text_data_{filename}'
            os.mkdir(self.address)

        with open(f'{self.address}/{name}', 'w') as file:
            for line in self.raw_data_text:
                file.writelines(line + '\n')

    def cut_first_part(self):  # ok 28.11.2021
        """inner method: get rid off all css lines"""

        index = 0
        data = None
        for item in self.raw_data_text:
            if item == '':
                data = self.raw_data_text[index + 1:]

            index += 1
        self.raw_data_text = data

    def cut_last_part(self):  # ok 28.11.2021
        """inner method: erases the "Created with Battlescribe" line"""

        end_index = 0
        for line in self.raw_data_text:
            if 'Created with' in line:
                end_index = self.raw_data_text.index(line)

        data = self.raw_data_text[:end_index]
        self.raw_data_text = data

    def cut_mid_size_tags_lines(self):  # ok 28.11.2021
        """inner method: erases the lines under lenth 12"""

        data = []

        for line in self.raw_data_text:
            if len(line) > 12:
                data.append(line)
        self.raw_data_text = data

    def mark_start_of_unit(self):  # ok 28.11.2021
        """inner method: add lines with marks where the units starts"""

        stop_str = '
'
        new_raw_data = []
        index = 0

        for line in self.raw_data_text:

            if index < len(self.raw_data_text) - 1:
                next_line = self.raw_data_text[index + 1]
                if stop_str in line and 'PL' in next_line and 'Battle Size [' not in next_line:
                    line = '__START OF A UNIT__'
            new_raw_data.append(line)
            index += 1
        self.raw_data_text = new_raw_data

    def deal_with_small_tags(self):  # ok 28.11.2021
        """inner method: exchanges table tags for spaces"""

        raw_data = []
        for line in self.raw_data_text:
            if '	' in line:
                line = line.replace('	', ' ')
            if '	' in line:
                line = line.replace('	', ' ')
            new_line = self.erase_tags_from_line(line)
            raw_data.append(new_line)

        self.raw_data_text = raw_data

    def erase_Categories_Configuration_lines(self):
        """inner method: erases lines that contain the word "Categories" & "Configuration" """

        raw_data = []
        for line in self.raw_data_text:
            if 'Categories: Configuration' not in line:
                raw_data.append(line)
        self.raw_data_text = raw_data

    def erase_Detachment_Command_Cost_lines(self):
        """inner method: erases lines that contain the word "Detachment Command Cost" """

        raw_data = []
        for line in self.raw_data_text:
            if 'Detachment Command Cost' not in line:
                raw_data.append(line)
        self.raw_data_text = raw_data

    def erase_configuration_CP_line(self):
        """inner method: erases lines that contain the word "configuration CP" """
        raw_data = []
        for line in self.raw_data_text:
            if 'Configuration [' not in line:
                raw_data.append(line)
        self.raw_data_text = raw_data

    def erase_blank_lines(self, data=None):
        """inner method: erases lines that contain newline signs and empty strs """
        if not data:
            data = self.raw_data_text
        raw_data = []
        for line in data:
            new_line = line.split('\n')
            if new_line[0]:
                raw_data.append(new_line[0])
        self.raw_data_text = raw_data

    def emplace_more_les_than_signs(self):
        """inner method: replaces code for right simbols"""

        new_data = []
        for line in self.raw_data_text:
            if MORE_THAN in line and LESS_THAN in line:
                line = self.more_les_than(line)

            new_data.append(line)
        self.raw_data_text = new_data

    def emplace_ampersand(self):
        """inner method: replaces code for right simbols"""

        new_data = []
        for line in self.raw_data_text:
            if AMPERSAND in line:
                line = self.ampersand(line)

            new_data.append(line)
        self.raw_data_text = new_data

    def emplace_apostrofe(self):
        """inner method: replaces code for right simbols"""

        new_data = []
        for line in self.raw_data_text:
            if APOSTROFE in line:
                line = self.apostrofe(line)

            new_data.append(line)
        self.raw_data_text = new_data

    def emplace_asterisco(self):
        """inner method: replaces code for right simbols"""

        new_data = []
        for line in self.raw_data_text:
            if ASTERISCO in line:
                # print(line)
                line = self.asterisco(line)
                # print(line)
            new_data.append(line)
        self.raw_data_text = new_data

    def populate_general_rules_data(self):
        """inner method: splits the raw text in general rules and unit rules to be parsed separately
        it modifies the raw text file"""

        index = 0
        new_raw_data = []

        while index < len(self.raw_data_text) - 1:
            line = self.raw_data_text[index]
            # get the top rules
            self.general_rules_text.append(line)

            # get the units data
            if 'No Force Org Slot [' in line:

                while 'Force Rules' not in line and index < len(self.raw_data_text) - 1:
                    line = self.raw_data_text[index]
                    if 'Force Rules' in line:
                        break
                    new_raw_data.append(line)
                    index += 1

            if 'HQ [' in line:
                while 'Force Rules' not in line and index < len(self.raw_data_text) - 1:
                    line = self.raw_data_text[index]
                    # self.raw_units_text.append(line)
                    if 'Force Rules' in line:
                        break
                    new_raw_data.append(line)

                    index += 1
            index += 1

        self.raw_units_text = new_raw_data

        # check if resume
        is_resume = False
        for line in self.raw_data_text:
            if 'Force Rules' in line or 'Selection Rules' in line:
                is_resume = True

        if is_resume:
            index = len(self.raw_data_text) - 1
            while index > 0:
                line = self.raw_data_text[index]
                if 'Force Rules' in line or 'Selection Rules' in line:
                    butt = self.raw_data_text[index:]
                    self.general_rules_text.extend(butt)
                    break
                index -= 1

    # GENERAL DATA GETTERS
    def get_army_name(self):  # ok 29.11.2021
        line = self.general_rules_text[0]
        self.name = line.split(' (')[0]
        self.general_rules_text.remove(line)

    def get_army_pts(self):  # ok 29.11.2021
        line = self.general_rules_text[0]
        pts_text = ''
        if 'CP, ' in line:
            a = line.split('CP, ')[1]
            pts_text = a.split(']')[0]

        elif ', , ' in line:
            a = line.split(', , ')[1]
            pts_text = a.split('pts]')[0]

        no_coma_pts = ''
        for char in pts_text:
            if char.isdigit():
                no_coma_pts += char
        self.pts = int(no_coma_pts)

    def get_detachment(self):  # ok 29.11.2022
        for line in self.general_rules_text:
            if 'Detachment' in line:
                self.detachment = line.split(' Detachment')[0]
                break

    def get_faction(self):
        for line in self.general_rules_text:

            if 'Detachment' in line and '(' in line:
                a = line.split('(')[1]
                faction = a.split(')')[0]
                self.faction = faction
                self.general_rules_text.remove(line)
                break

    def get_CP(self):  # ok 29.11.2021
        for line in self.general_rules_text:

            if 'Battle Size' in line:

                if '[' in line:
                    cp = line.split('[')[1]
                    cp = cp.split('CP]')[0]
                    self.CP = int(cp)
                    self.general_rules_text.remove(line)
                    break
                else:
                    print('CPs not specified in list')
                    print('I asume is 18CP')
                    self.CP = 18
                    self.general_rules_text.remove(line)
                    break

        for line in self.general_rules_text:
            if 'Configuration [' in line:
                cp = line.split('[')[1]
                cp = cp.split('CP]')[0]
                self.CP = int(cp)
                self.general_rules_text.remove(line)
                break

    def get_battle_type(self):  # ok 29.11.2021
        for line in self.general_rules_text:

            if 'Selection' in line and 'PL' in line:
                if '. ' in line:
                    a = line.split('. ')[1]
                    battle_size = a.split(' (')[0]
                    self.battle_type = battle_size
                    self.general_rules_text.remove(line)

    def get_first_line_rule(self):
        for line in self.general_rules_text:
            if 'Rules: ' in line:
                rules = line.split('Rules: ')[1]
                self.army_rules_names.append(rules)
                self.general_rules_text.remove(line)

    def get_sub_faction(self):
        subfaction_name = ''
        subfaction = ''

        for line in self.general_rules_text:

            if ' Choice' in line:
                subfaction_name = line.split(' Choice')[0]
                self.general_rules_text.remove(line)
                break
            if 'Selector' in line:
                subfaction_name = line.split('Selector')[0]
                subfaction_name = subfaction_name.split('**')[1]
                self.general_rules_text.remove(line)
                break
            else:
                subfaction_name = line.strip()
                self.general_rules_text.remove(line)
                break

        if ' / ' in subfaction_name:
            subfaction_name = subfaction_name.split(' / ')[0]

        for line in self.general_rules_text:
            if 'Selections:' in line:
                if 'Custom' in line:
                    subfaction = self.fix_succesor_chapter_order(line)
                    self.general_rules_text.remove(line)
                    break
                if 'Selections:' in line:
                    subfaction = line.split('Selections: ')[1]
                    self.general_rules_text.remove(line)
                    break

        self.subfaction = (subfaction_name, subfaction)

    def get_faction_ability(self):
        for line in self.general_rules_text:
            if 'Abilities: ' in line:
                faction_ability = line.split('Abilities: ')[1]
                if ', ' in faction_ability:
                    faction_ability = faction_ability.split(', ')
                    for item in faction_ability:
                        self.army_rules_names.append(item)
                    self.general_rules_text.remove(line)
                    break
                else:
                    self.army_rules_names.append(faction_ability)
                    self.general_rules_text.remove(line)
                    break

    @staticmethod
    def fix_succesor_chapter_order(line):
        """inner method: in the case of the marines, the sucessor chapter rules are not ordered, this function fixes
        that """
        new_line = ''
        arrange = []
        line_list = line.split(':')[1]
        line_list = line_list.split(',')

        # put succesor first in list
        for item in line_list:
            if 'Successor' in item:
                arrange.append(item)

        for item in line_list:
            if ' Custom Chapter' in item:
                arrange.append(item)

        for item in arrange:
            if item != arrange[-1]:
                new_line += item + ', '
            else:
                new_line += item
        return new_line

    def erase_pages_and_useless_data(self):
        """inner method: find rest of useless data to erase"""
        for line in self.general_rules_text:
            if self.condition1(line) and self.condition2(line):
                self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            if line == 'Description':
                self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            if 'Selection Rules' in line:
                self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            if 'Force Rules' in line:
                self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            if 'Codex:' in line:
                step1 = line.split('Codex: ')[1]
                self.codex = step1.split(' p')[0]
                self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            for book_reference in self.book_references_list:
                if book_reference in line:
                    self.general_rules_text.remove(line)

        for line in self.general_rules_text:
            if 'This unit has the following abilities:' in line:
                self.general_rules_text.remove(line)

    @staticmethod
    def condition1(line):
        is_useless = False
        useless_data = ['[', '(', 'Abilities', 'Description', 'Selection Rules']
        for item in useless_data:
            if item in line:
                is_useless = True
        return is_useless

    @staticmethod
    def condition2(line):
        return len(line) < 40

    def fix_army_rules_names(self):

        new_list = []
        for item in self.army_rules_names:
            if ',' in item:
                subitems = item.split(', ')
                for sub_item in subitems:
                    new_list.append(sub_item.strip())
            else:
                new_list.append(item.strip())

        self.army_rules_names = new_list

    def get_the_rest_of_the_general_rules(self):

        for line in self.general_rules_text:
            if ':' in line and len(line) < 40:
                rule = line.split(':')[0]
                if rule not in self.army_rules_names:
                    self.army_rules_names.append(rule)

    def make_general_rules_text_pair_odd_lines(self):

        index = 0
        max_index = len(self.general_rules_text) - 1

        while index < max_index:
            current_line = self.general_rules_text[index]
            if 'Angels of Death' in current_line:
                self.general_rules_text.insert(index + 1, ANGELS_OF_DEATH)
            elif len(current_line) < 35:
                next_line = self.general_rules_text[index + 1]
                if len(next_line) < 35:
                    self.general_rules_text.insert(index + 1, 'NO DATA')
                    index += 1
            index += 1

    def get_the_damned_rules(self):

        self.general_rules_text = self.set_in_same_order(self.general_rules_text)

        new_dict = {}
        new_list = []
        index = 0
        max_index = len(self.general_rules_text) - 1
        key = ''
        value = ''

        while index < max_index:
            line = self.general_rules_text[index]
            if len(line) < 35:  # found new key
                if key and value:  # last key value is over, needs to record
                    new_dict[key] = value
                    new_list.append([key, value])
                # get new key
                key = line
                # reset value
                if value:
                    value = ''
                index += 1
            else:
                value += line
                index += 1
            if index == max_index:  # ends the text
                if not value:
                    value += self.general_rules_text[-1]
                new_dict[key] = value
                new_list.append([key, value])

        self.army_rules = new_dict
        self.army_rules_list = new_list

    def record_rules_in_csv(self):

        file_name = f'general rules {self.name}'
        fieldnames = ['Ability', 'Description']

        with open(f'{self.full_path}{self.name}/{file_name}.csv', 'w') as f:
            writer = csv.writer(f)

            writer.writerow(fieldnames)

            for item in self.army_rules_list:
                writer.writerow(item)

    # TOOLS
    @staticmethod
    def is_new_line_sign(line):
        if '\n' in line:
            print(line.count('\n'))

    @staticmethod
    def erase_tags_from_line(line):

        index = 0
        new_line = ''
        while index < len(line):
            if line[index] == '<':
                while line[index] != '>':
                    index += 1
            else:
                if line[index] != '>':
                    new_line += line[index]
                index += 1

        return new_line

    @staticmethod
    def set_in_same_order(some_list):

        unique_items = []

        for item in some_list:
            if item not in unique_items:
                unique_items.append(item)

        return unique_items

    @staticmethod
    def ampersand(line):
        """this is made to deal with 0ne str at the time"""

        if type(line) == list:
            for item in line:
                if AMPERSAND in item:
                    new_line = item.replace(AMPERSAND, '&')
                    return new_line
        elif type(line) == str:
            if AMPERSAND in line:
                new_line = line.replace(AMPERSAND, '&')
                return new_line

        return AMPERSAND in line

    @staticmethod
    def more_les_than(line):

        if type(line) == list:
            for item in line:
                if LESS_THAN in item and MORE_THAN in item:
                    new_line = item.replace(LESS_THAN, '<')
                    new_line = new_line.replace(MORE_THAN, '>')
                    return new_line
        elif type(line) == str:
            if LESS_THAN in line and MORE_THAN in line:
                new_line = line.replace(LESS_THAN, '<')
                new_line = new_line.replace(MORE_THAN, '>')
                return new_line

        return LESS_THAN in line and MORE_THAN in line

    @staticmethod
    def apostrofe(line):

        if type(line) == list:
            for item in line:
                if APOSTROFE in item:
                    new_line = item.replace(APOSTROFE, "'")
                    return new_line
        elif type(line) == str:
            if APOSTROFE in line:
                new_line = line.replace(APOSTROFE, "'")
                return new_line

        return APOSTROFE in line

    @staticmethod
    def asterisco(line):

        if type(line) == list:
            for item in line:
                if ASTERISCO in item:
                    new_line = item.replace(ASTERISCO, "*")
                    return new_line
        elif type(line) == str:
            if ASTERISCO in line:
                new_line = line.replace(ASTERISCO, "*")
                return new_line

        return ASTERISCO in line

    @staticmethod
    def is_item_of_list_in_line(line, list_of_items):

        for item in list_of_items:
            if item in line:
                return True
        return False

    @staticmethod
    def wich_item_is_in_line(line, list_of_items, return_index=False):

        if return_index:
            index = 0
            for item in list_of_items:
                if item in line:
                    return list_of_items.index(item)
                index += 1
            return None  # it returns None automatically

        else:
            for item in list_of_items:
                if item in line:
                    return item
            return None

    @staticmethod
    def move_next_line(index, data):

        index += 1
        line = data[index]
        return index, line

    @staticmethod
    def is_quote_or_dash(line):

        is_quote_or_dash = False

        if line[0].isdigit():
            for char in line:
                if char == '"':
                    is_quote_or_dash = True
                    break

        elif line[0] == '-':
            is_quote_or_dash = True

        return is_quote_or_dash

    # PREPARE UNIT TEXT DATA
    def clean_unit_pages_references(self):
        for line in self.raw_units_text:
            if ':' in line:
                for book in RULE_BOOKS:
                    if book in line:
                        self.raw_units_text.remove(line)

    def points_per_role(self):

        pts_list = []
        data = self.raw_units_text

        for line in data:
            is_role = self.is_item_of_list_in_line(line, ROLES)
            if is_role and ' [' in line:
                # role = self.wich_item_is_in_line(line, ROLES)
                step = line.split(', ')[1]
                pts = step.split('pts')[0]
                pts_list.append(pts)
                self.raw_units_text.remove(line)

        return pts_list

    def how_many_each_roles_in_army(self):
        index = 0
        max_index = len(self.raw_units_text) - 1
        new_list = []
        amounts = []
        data = self.raw_units_text
        line = data[index]
        role = None

        while index < max_index:
            is_role = self.wich_item_is_in_line(line, ROLES)

            if is_role and ' [' in line:  # found a role
                index, line = self.move_next_line(index, data)
                role = is_role

            elif '__START OF A UNIT__' in line and index < max_index:
                index, line = self.move_next_line(index, data)
                if role:
                    # print(role)
                    new_list.append(role)

            else:
                if index < max_index:
                    index, line = self.move_next_line(index, data)

        all_roles = self.set_in_same_order(new_list)

        for item in all_roles:
            amount = new_list.count(item)
            amounts.append(amount)

        self.amount_of_units = functools.reduce(lambda a, b: a + b, amounts)
        pts = self.points_per_role()
        roles_amount_pts = list(zip(all_roles, amounts, pts))
        self.roles = roles_amount_pts

        # create a stop '__START OF A UNIT__' for unit chunk parse
        self.raw_units_text.append('__START OF A UNIT__')

    # Psy powers parse and clean up
    def get_psychic_powers(self):

        # for line in self.raw_units_text:
        #     print(line)

        data = self.raw_units_text
        index = 0
        max_index = len(data) - 1
        new_raw_text = []
        psy_chunk = []

        while index < max_index:
            line = data[index]
            if 'Warp Charge Range Details' in line:
                index, line = self.move_next_line(index, data)
                while 'Cast Deny Powers Known Other' not in line:
                    if line == 'Psyker':
                        index, line = self.move_next_line(index, data)
                    else:
                        if 'If' in line:
                            begining = line[:6]
                            index_start = line.index('If')
                            line = begining + line[index_start:]
                        psy_chunk.append(line)
                        psy_chunk = self.set_in_same_order(psy_chunk)
                        index, line = self.move_next_line(index, data)
            else:
                new_raw_text.append(line)
                index += 1

        self.raw_psy_text = psy_chunk

    def parse_psy_powers(self):
        data = self.raw_psy_text
        index = 0
        max_index = len(data) - 1
        details = None
        name = None
        warp_charge = None
        pow_range = None

        # for line in self.raw_psy_text:
        #     print(line)

        while index < max_index:
            line = data[index]
            if index % 2 == 1:
                value = line
                # print(value)
                warp_charge = value[0]
                pow_range = value[2:5]
                pow_range = pow_range.split('"')[0]
                details = value[6:]
                index += 1
            else:
                if ')' in line:
                    line = line.split(') ')[1]
                if ' (' in line:
                    line = line.split(' (')[0]
                name = line
                index += 1
            if name and warp_charge and warp_charge and pow_range and details:
                self.psychic_powers.append([name, warp_charge, pow_range, details])
                name = None
                warp_charge = None
                pow_range = None
                details = None

        self.raw_psy_text = []

    def record_psypow_in_csv(self):
        # todo:
        file_name = f'Psy Powers {self.name}'
        fieldnames = PSY_POW_STATS

        with open(f'{self.address}/{file_name}.csv', 'w') as f:
            writer = csv.writer(f)

            writer.writerow(fieldnames)

            for item in self.psychic_powers:
                writer.writerow(item)

    def parse_weapon_stats_record_in_csv(self):
        for item in self.all_unit_chunks:
            print(item)
        pass

    def parse_models_stats_record_in_csv(self):
        pass

    # unit parse and clean up
    # def build_a_unit(self):
    #
    #     # TODO: find all inf build model by model first
    #
    #     fieldrole = ''
    #     name = ''
    #     models = []  # ['sgt', 'marine', 'marine' ...]
    #     categories = []  # list of names of categories
    #     equipments = []  # list of equipment names
    #     abilities = []  # list of abilities names
    #
    #     is_psyker = False
    #
    #     army = self.name
    #     stats = None
    #
    #     # THIS DATA NEEDS TO BE RECORDED INDIVIDUALY TO CSV DATABASE
    #     # TODO: Create model object with each model data
    #     model = [army, fieldrole, name, stats, categories, equipments, abilities]  # unit de tuple ('M', 6)...
    #     for category in categories:
    #         if category == 'Psyker':
    #             is_psyker = True
    #
    #     if is_psyker:
    #         cast = 2
    #         deny = 1
    #         power_known = 'Smite'
    #         other_powers = []
    #         psyker_model = [cast, deny, power_known, other_powers]
    #
    #         model.extend(psyker_model)
    #     # TODO: Add rules to the database
    #     rules = []  # pass this rules to the general rule pool
    #     # TODO: Create weapon objects
    #     weapon_stats = []
    #
    #     # TODO: Create a Unit object with all this models objects
    #     unit = [fieldrole, name, models, categories, equipments, abilities]
    #
    # def get_id(self):
    #     self.serial_No += 1
    #     return self.serial_No
    #
    # def parse_unit_name_pts_pl_erase_line(self, line):
    #     if '__START OF A UNIT__' in line:
    #
    #         index = self.raw_units_text.index(line)
    #         # erase_from = index
    #
    #         # moving to Name & pts line
    #         index, line = self.move_next_line(index, self.raw_units_text)
    #         unit_name = line.split(' [')[0]
    #         pl_step, pts_step = line.split(' PL, ')
    #         unit_PL = pl_step.split('[')[1]
    #         unit_pts = pts_step.split('pts]')[0]
    #         if 'CP, ' in unit_pts:
    #             unit_pts = unit_pts.split('CP, ')[1]
    #
    #         # moving to selection line
    #         index, line = self.move_next_line(index, self.raw_units_text)
    #         selection = line[12:]
    #         # print(line)
    #
    #         # fractioned_line = line.split(', ')
    #         # print(fractioned_line)
    #
    #         return unit_name, int(unit_pts), int(unit_PL)

    def split_unit_chunks(self):

        data = self.raw_units_text

        start_index = 0
        mid_index = 0
        # max_index = len(data) - 1
        chunk = []
        has_started = False

        for line in data:
            if '__START OF A UNIT__' in line:
                start_index = data.index(line)
                has_started = True
            else:
                if has_started:
                    chunk.append(line)
                    mid_index = start_index + 1

            if '__START OF A UNIT__' in line and mid_index > start_index:
                self.all_unit_chunks.append(chunk)
                self.raw_units_text.remove(line)

                for item in chunk:
                    if item in data:
                        self.raw_units_text.remove(item)
                break

    # todo: make this block more efective?
    @staticmethod
    def get_unit_name(unit):

        line_index = 0
        name = ''

        for line in unit:
            # print(line)
            if line_index == 0:
                if ' [Legends] [' in unit[line_index]:
                    name = line.split(' [Legends] [')[0]
                else:
                    name = unit[line_index].split(' [')[0]

        return name

    @staticmethod
    def get_unit_pl(unit):
        line_index = 0
        pl = None

        for line in unit:
            if ' [Legends] [' in unit[line_index]:
                _, line = line.split(' [Legends] [')

            else:
                _, line = unit[line_index].split(' [')

            pl = int(line.split(' PL')[0])

            break
        return pl

    @staticmethod
    def get_unit_cp(unit):

        cp = None

        for line in unit:
            if 'CP, ' in line:
                cp, _ = line.split('CP, ')
                cp = int(cp.split('PL, ')[1])
            break
        return cp

    @staticmethod
    def get_unit_pts(unit):

        pts = None

        for line in unit:
            line = line.split('PL, ')[1]
            if 'CP, ' in line:
                line = line.split('CP, ')[1]
            pts = int(line.split('pts]')[0])
            break
        return pts

    @staticmethod
    def get_unit_weapons(unit):

        weapons = []

        for line in unit:
            if 'Selections:' in line:
                print(line)
            if ', Weapon: ' in line:
                print(line)
            #     line, weapon = line.split(', Weapon: ')
            #     weapons = weapon.split(', ')

        return weapons

    @staticmethod
    def get_unit_models_in_unit(unit):

        models_in_unit = None

        for line in unit:
            if 'Unit: ' in line:
                line, models_in_unit = line.split(', Unit: ')

                if models_in_unit and '[1]' not in line:

                    if ',' in models_in_unit:
                        if ', Warlord Trait: ' in models_in_unit:
                            models_in_unit, warlord_trait = models_in_unit.split(', Warlord Trait: ')
                        else:
                            models_in_unit = models_in_unit.split(', ')

        return models_in_unit

    @staticmethod
    def get_unit_warlord_trait(unit):

        warlord_trait = None

        for line in unit:
            if 'Unit: ' in line:
                line, models_in_unit = line.split(', Unit: ')

                if models_in_unit and '[1]' not in line:

                    if ',' in models_in_unit:
                        if ', Warlord Trait: ' in models_in_unit:
                            _, warlord_trait = models_in_unit.split(', Warlord Trait: ')

        return warlord_trait

    @staticmethod
    def get_unit_categories(unit):
        categories = None
        nueva_lista = []

        for line in unit:
            if 'Categories:' in line:
                a = line.split('Categories: ')[1]
                categories = a.split('Faction: ')

        if categories:
            for item in categories:
                if item:
                    # print(item.split(', '))
                    sub_item = item.split(', ')
                    for fraction in sub_item:
                        if fraction:
                            # print(fraction)
                            nueva_lista.append(fraction)

        categories = nueva_lista
        return categories

    @staticmethod
    def get_unit_wound_track(unit):

        for line in unit:
            if 'wounds' in line or 'Wound Track: ' in line or 'wounds' in line or '[1]' in line:
                wound_track = line
                # models_in_unit = line  # ???

    @staticmethod
    def get_unit_is_psyker(unit):

        is_psyker = False

        for line in unit:
            if ' Psyker: ' in line:
                # lista = line.split(', Psyker: ')[0]
                is_psyker = True

        return is_psyker

    @staticmethod
    def get_unit_psy_pow(unit):
        psy_pow = []

        for _ in unit:

            if ', Psychic Power: ' in unit:
                lista, psy_pow = unit.split(', Psychic Power: ')

        return psy_pow

    @staticmethod
    def get_unit_abilities(unit):

        abilities = []

        for line in unit:
            abilities = line.split('Abilities: ')[1]
            if ', ' in abilities:
                abilities = abilities.split(', ')

        return abilities

    def take_rules_from_units(self, unit):
        """here the 'unit' argument comes forlooped from 'test' func"""

        line_index = 0

        unit_list = []

        # name = self.get_unit_name(unit)
        # unit_list.append(name)
        # print(name)
        # pl = self.get_unit_pl(unit)
        # unit_list.append(pl)
        # cp = self.get_unit_cp(unit)
        # unit_list.append(cp)
        # pts = self.get_unit_pts(unit)
        # unit_list.append(pts)
        # categories = self.get_unit_categories(unit)
        # unit_list.append(categories)

        # weapons = self.get_unit_weapons(unit)
        # print(weapons)

        # warlord_trait = self.get_unit_warlord_trait(unit)
        # print(warlord_trait)
        # models_in_unit = self.get_unit_models_in_unit(unit)
        # print(models_in_unit)
        # abilities = self.get_unit_abilities(unit)
        # print(abilities)
        # is_psyker = self.get_unit_is_psyker(unit)
        # print(is_psyker)
        # psy_pow = self.get_unit_psy_pow(unit)
        # print(psy_pow)

        model_and_stats = self.model_stats(unit)
        # print(next(model_and_stats))

        # weapon_stats = self.weapon_stats(unit)
        index = 0
        max_index = len(unit) - 1

        description = []
        unit_data = []

        while index <= max_index:
            line = unit[index]
            # print(line)
            if line == 'Description' or line == 'Abilities' or line == 'Unit':
                pass

            else:
                unit_data.append(line)

            if line == 'Description' or len(line) < 10 and 'Unit' in line:
                while index < max_index:
                    index += 1
                    line = unit[index]
                    description.append(line)

                    # inside description there are stats for Unit, psyker, weapon. parse separatly

            index += 1
        #
        # for line in unit:
        #     print(line)

    @staticmethod
    def fix_weapon_rules(fractions):  # ['60"', 'Heavy', '4D3', '4', '0', '1', 'This', 'weapon', 'can', 'target', ...
        max_index = len(fractions) - 1
        rule_start_index = 6
        fixed_rules = ''

        if len(fractions) > 9:
            if not fractions[rule_start_index].isdigit():
                while rule_start_index <= max_index:
                    fixed_rules += fractions[rule_start_index]
                    rule_start_index += 1
                    if rule_start_index <= max_index:
                        fixed_rules += ' '
        if fixed_rules:
            new_stats = fractions[:6]
            new_stats.append(fixed_rules)
            return new_stats
        return fractions

    @staticmethod
    def fix_weapon_type(fractions):
        for fraction in fractions:
            if fraction == 'Rapid':
                at_index = fractions.index('Rapid')
                fractions.remove(fraction)
                fraction = 'Rapid Fire'
                fractions.insert(at_index, fraction)

            if fraction == 'Fire':
                fractions.remove(fraction)
        return fractions

    @staticmethod
    def model_stats(unit):
        index = 0
        max_index = len(unit) - 1
        models = []

        while index <= max_index:
            model_chunk = []
            line = unit[index]
                                                    # - M S A ends at # [4 PL, 85pts] or unit
            if line == 'M WS BS S T W A Ld Save':   # Razorback [3] (1-2 wounds remaining) 3" 6+ 5+ 6 7 N/A 1 8 3+
                                                    # f'{name} [counter] ({wounds} wounds remaining) * 6+ * 6 7 10 * 8 3+' exchange
                                                    # * 6+ * 6 7 10 * 8 3+
                                                    # Chaos Rhino3  1-2 3" 5+ 1

                index += 1

                while index <= max_index:
                    line = unit[index]
                    if line == 'Weapon':
                        break
                    model_chunk.append(line)
                    index += 1

            if model_chunk:
                # print(model_chunk)
                yield model_chunk
                # models.append(model_chunk)

            index += 1

        # return models

    def weapon_stats(self, unit):

        index = 0
        max_index = len(unit) - 1

        weapon = []

        while index <= max_index:
            line = unit[index]
            print(line)
            if 'Unit: ' in line and line[:4] == 'Unit' and '[1]' in line:
                # print(line[6:41])  # Whirlwind [1] (6+ wounds remaining)
                pass
            if line == 'Unit':
                index, line = self.move_next_line(index, unit)
                index, line = self.move_next_line(index, unit)
                # print(line)  # models name
                # while index <= max_index:
                #     index, line = self.move_next_line(index, unit)

                # if line[0] != '-' or not line[0].isdigit():
                #     break
            if line == 'Weapon':
                pass
            if self.is_quote_or_dash(line):
                fractions = line.split(' ')
                fractions = self.fix_weapon_rules(fractions)
                fraction = self.fix_weapon_type(fractions)

                # print(fractions, len(fractions)) #stats
                # index, line = self.move_next_line(index, unit)

                # print(line)

            index += 1

        return weapon

    def test(self):

        # units_headings = ['Name', 'Pts', 'PL']

        # self.unit_chunks()

        for _ in range(0, self.amount_of_units):
            self.split_unit_chunks()

        # print()
        # print(f'{self.name}')
        # print('↓ units of below army ↓')

        "↓ turn this on"
        for unit_chunk in self.all_unit_chunks:
            for line in unit_chunk:
                print(line)
                pass
            self.take_rules_from_units(unit_chunk)

        # units = []
        # for line in self.raw_units_text:
        #     unit = self.parse_unit_name_pts_pl_erase_line(line)
        #
        #     units.append(unit)
        # todo: fix error (missing units in self.raw_unit_text)

if __name__ == '__main__':
    directory = './files_in_html'

    cajlito = f'{directory}/Cajlito (HTML).html'
    machete = f'{directory}/1000pts test tau (HTML).html'
    machete2 = f'{directory}/1000pts las perras test (HTML).html'
    victor = f'{directory}/1000 BloodClaws rush (HTML).html'
    victor2 = f'{directory}/1000pts thunder rush (HTML).html'
    victor3 = f'{directory}/1000 jump rush (HTML).html'

    angels = f'{directory}/500pts DA green wing (HTML).html'
    orks = f'{directory}/4000 Orks test (HTML).html'
    orks2 = f'{directory}/6000 Orks test (HTML).html'
    tyranids = f'{directory}/1000pts Tyranids test (HTML).html'
    necrons = f'{directory}/1000pts necrons test (HTML).html'
    astra = f'{directory}/1000 pts astra militarum test (HTML).html'
    astra2 = f'{directory}/500 astra (HTML).html'
    smurf = f'{directory}/1500 Smurfs test (HTML).html'
    scars = f'{directory}/500pts white scars (HTML).html'

    all_army_list = [cajlito, machete, machete2, victor, victor2, victor3, angels, orks, orks2, tyranids, necrons,
                     astra, astra2, smurf, scars]

    data_colected_so_far = ['name', 'pts', 'CP', 'battle_type', 'detachment', 'faction', 'subfaction', 'codex',
                            'amount_of_units', 'roles', 'psychic_powers', 'units']

    # for item_ in all_army_list:
    #     try:
    #         current_army = ParseFromHtml(item_)
    #     #         print(current_army)
    #     except IndexError as e:
    #         print(f" IndexError {e}: {item_.split('/')[-1]}")
    #     except Exception as e:
    #         print(f" {Exception} {e}: {item_.split('/')[-1]}")

    current_army = ParseFromHtml(cajlito)

    # for item_ in all_army_list:
    #     current_army = ParseFromHtml(item_)
    # print(current_army)


"""Consumer - fetching messages and managing consumer state.

The Consumer is responsible for:

   - Holds reference to the transport that created it

   - ... and the app via ``self.transport.app``.

   - Has a callback that usually points back to ``Conductor.on_message``.

   - Receives messages and calls the callback for every message received.

   - Keeps track of the message and its acked/unacked status.

   - The Conductor forwards the message to all Streams that subscribes
     to the topic the message was sent to.

       + Messages are reference counted, and the Conductor increases
         the reference count to the number of subscribed streams.

       + ``Stream.__aiter__`` is set up in a way such that when what is
         iterating over the stream is finished with the message, a
         finally: block will decrease the reference count by one.

       + When the reference count for a message hits zero, the stream will
         call ``Consumer.ack(message)``, which will mark that topic +
         partition + offset combination as "committable"

       + If all the streams share the same key_type/value_type,
         the conductor will only deserialize the payload once.

   - Commits the offset at an interval

      + The Consumer has a background thread that periodically commits the
        offset.

      - If the consumer marked an offset as committable this thread
        will advance the committed offset.

      + To find the offset that it can safely advance to the commit thread
        will traverse the _acked mapping of TP to list of acked offsets, by
        finding a range of consecutive acked offsets (see note in
        _new_offset).

"""
import abc
import asyncio
import gc
import typing

from collections import defaultdict
from time import monotonic
from typing import (
    Any,
    AsyncIterator,
    Awaitable,
    ClassVar,
    Dict,
    Iterable,
    Iterator,
    List,
    Mapping,
    MutableMapping,
    MutableSet,
    NamedTuple,
    Optional,
    Set,
    Tuple,
    Type,
    cast,
)
from weakref import WeakSet

from mode import Service, ServiceT, flight_recorder, get_logger
from mode.threads import MethodQueue, QueueServiceThread
from mode.utils.futures import notify
from mode.utils.locks import Event
from mode.utils.text import pluralize
from mode.utils.times import Seconds
from faust.exceptions import ProducerSendError
from faust.types import AppT, ConsumerMessage, Message, RecordMetadata, TP
from faust.types.core import HeadersArg
from faust.types.transports import (
    ConsumerCallback,
    ConsumerT,
    PartitionsAssignedCallback,
    PartitionsRevokedCallback,
    ProducerT,
    TPorTopicSet,
    TransactionManagerT,
    TransportT,
)
from faust.types.tuples import FutureMessage
from faust.utils import terminal
from faust.utils.functional import consecutive_numbers
from faust.utils.tracing import traced_from_parent_span

if typing.TYPE_CHECKING:  # pragma: no cover
    from faust.app import App as _App
else:
    class _App: ...  # noqa: E701

__all__ = ['Consumer', 'Fetcher']

# These flags are used for Service.diag, tracking what the consumer
# service is currently doing.
CONSUMER_FETCHING = 'FETCHING'
CONSUMER_PARTITIONS_REVOKED = 'PARTITIONS_REVOKED'
CONSUMER_PARTITIONS_ASSIGNED = 'PARTITIONS_ASSIGNED'
CONSUMER_COMMITTING = 'COMMITTING'
CONSUMER_SEEKING = 'SEEKING'
CONSUMER_WAIT_EMPTY = 'WAIT_EMPTY'

logger = get_logger(__name__)

RecordMap = Mapping[TP, List[Any]]

class TopicPartitionGroup(NamedTuple):
    """Tuple of ``(topic, partition, group)``."""

    topic: str
    partition: int
    group: int

def ensure_TP(tp: Any) -> TP:
    """Convert aiokafka ``TopicPartition`` to Faust ``TP``."""
    return tp if isinstance(tp, TP) else TP(tp.topic, tp.partition)

def ensure_TPset(tps: Iterable[Any]) -> Set[TP]:
    """Convert set of aiokafka ``TopicPartition`` to Faust ``TP``."""
    return {ensure_TP(tp) for tp in tps}

class Fetcher(Service):
    """Service fetching messages from Kafka."""

    app: AppT

    logger = logger
    _drainer: Optional[asyncio.Future] = None

    def __init__(self, app: AppT, **kwargs: Any) -> None:
        self.app = app
        super().__init__(**kwargs)

    async def on_stop(self) -> None:
        """Call when the fetcher is stopping."""
        if self._drainer is not None and not self._drainer.done():
            self._drainer.cancel()
            while True:
                try:
                    await asyncio.wait_for(self._drainer, timeout=1.0)
                except StopIteration:
                    # Task is cancelled right before coro stops.
                    break
                except asyncio.CancelledError:
                    break
                except asyncio.TimeoutError:
                    self.log.warning('Fetcher is ignoring cancel or slow :(')
                else:  # pragma: no cover
                    # coverage does not record this line as being executed
                    # but I've verified that it is [ask]
                    break

    @Service.task
    async def _fetcher(self) -> None:
        try:
            consumer = cast(Consumer, self.app.consumer)
            self._drainer = asyncio.ensure_future(
                consumer._drain_messages(self),
                loop=self.loop,
            )
            await self._drainer
        except asyncio.CancelledError:
            pass
        finally:
            self.set_shutdown()

class TransactionManager(Service, TransactionManagerT):
    """Manage producer transactions."""

    app: AppT

    transactional_id_format = '{group_id}-{tpg.group}-{tpg.partition}'

    def __init__(self, transport: TransportT,
                 *,
                 consumer: 'ConsumerT',
                 producer: 'ProducerT',
                 **kwargs: Any) -> None:
        self.transport = transport
        self.app = self.transport.app
        self.consumer = consumer
        self.producer = producer
        super().__init__(**kwargs)

    async def flush(self) -> None:
        """Wait for producer to transmit all pending messages."""
        await self.producer.flush()

    async def on_partitions_revoked(self, revoked: Set[TP]) -> None:
        """Call when the cluster is rebalancing and partitions are revoked."""
        await traced_from_parent_span()(self.flush)()

    async def on_rebalance(self,
                           assigned: Set[TP],
                           revoked: Set[TP],
                           newly_assigned: Set[TP]) -> None:
        """Call when the cluster is rebalancing."""
        T = traced_from_parent_span()
        # Stop producers for revoked partitions.
        revoked_tids = sorted(self._tps_to_transactional_ids(revoked))
        if revoked_tids:
            self.log.info(
                'Stopping %r transactional %s for %r revoked %s...',
                len(revoked_tids),
                pluralize(len(revoked_tids), 'producer'),
                len(revoked),
                pluralize(len(revoked), 'partition'))
            await T(self._stop_transactions, tids=revoked_tids)(revoked_tids)

        # Start produers for assigned partitions
        assigned_tids = sorted(self._tps_to_transactional_ids(assigned))
        if assigned_tids:
            self.log.info(
                'Starting %r transactional %s for %r assigned %s...',
                len(assigned_tids),
                pluralize(len(assigned_tids), 'producer'),
                len(assigned),
                pluralize(len(assigned), 'partition'))
            await T(self._start_transactions,
                    tids=assigned_tids)(assigned_tids)

    async def _stop_transactions(self, tids: Iterable[str]) -> None:
        T = traced_from_parent_span()
        producer = self.producer
        for transactional_id in tids:
            await T(producer.stop_transaction)(transactional_id)

    async def _start_transactions(self, tids: Iterable[str]) -> None:
        T = traced_from_parent_span()
        producer = self.producer
        for transactional_id in tids:
            await T(producer.maybe_begin_transaction)(transactional_id)

    def _tps_to_transactional_ids(self, tps: Set[TP]) -> Set[str]:
        return {
            self.transactional_id_format.format(
                tpg=tpg,
                group_id=self.app.conf.id,
            )
            for tpg in self._tps_to_active_tpgs(tps)
        }

    def _tps_to_active_tpgs(self, tps: Set[TP]) -> Set[TopicPartitionGroup]:
        assignor = self.app.assignor
        return {
            TopicPartitionGroup(
                tp.topic,
                tp.partition,
                assignor.group_for_topic(tp.topic),
            )
            for tp in tps
            if not assignor.is_standby(tp)
        }

    async def send(self, topic: str, key: Optional[bytes],
                   value: Optional[bytes],
                   partition: Optional[int],
                   timestamp: Optional[float],
                   headers: Optional[HeadersArg],
                   *,
                   transactional_id: str = None) -> Awaitable[RecordMetadata]:
        """Schedule message to be sent by producer."""
        group = transactional_id = None
        p = self.consumer.key_partition(topic, key, partition)
        if p is not None:
            group = self.app.assignor.group_for_topic(topic)
            transactional_id = f'{self.app.conf.id}-{group}-{p}'
        return await self.producer.send(
            topic, key, value, p, timestamp, headers,
            transactional_id=transactional_id,
        )

    def send_soon(self, fut: FutureMessage) -> None:
        raise NotImplementedError()

    async def send_and_wait(self, topic: str, key: Optional[bytes],
                            value: Optional[bytes],
                            partition: Optional[int],
                            timestamp: Optional[float],
                            headers: Optional[HeadersArg],
                            *,
                            transactional_id: str = None) -> RecordMetadata:
        """Send message and wait for it to be transmitted."""
        fut = await self.send(topic, key, value, partition, timestamp, headers)
        return await fut

    async def commit(self, offsets: Mapping[TP, int],
                     start_new_transaction: bool = True) -> bool:
        """Commit offsets for partitions."""
        producer = self.producer
        group_id = self.app.conf.id
        by_transactional_id: MutableMapping[str, MutableMapping[TP, int]]
        by_transactional_id = defaultdict(dict)

        for tp, offset in offsets.items():
            group = self.app.assignor.group_for_topic(tp.topic)
            transactional_id = f'{group_id}-{group}-{tp.partition}'
            by_transactional_id[transactional_id][tp] = offset

        if by_transactional_id:
            await producer.commit_transactions(
                by_transactional_id, group_id,
                start_new_transaction=start_new_transaction,
            )
        return True

    def key_partition(self, topic: str, key: bytes) -> TP:
        raise NotImplementedError()

    async def create_topic(self,
                           topic: str,
                           partitions: int,
                           replication: int,
                           *,
                           config: Mapping[str, Any] = None,
                           timeout: Seconds = 30.0,
                           retention: Seconds = None,
                           compacting: bool = None,
                           deleting: bool = None,
                           ensure_created: bool = False) -> None:
        """Create/declare topic on server."""
        return await self.producer.create_topic(
            topic, partitions, replication,
            config=config,
            timeout=timeout,
            retention=retention,
            compacting=compacting,
            deleting=deleting,
            ensure_created=ensure_created,
        )

    def supports_headers(self) -> bool:
        """Return :const:`True` if the Kafka server supports headers."""
        return self.producer.supports_headers()

class Consumer(Service, ConsumerT):
    """Base Consumer."""

    app: AppT

    logger = logger

    #: Tuple of exception types that may be raised when the
    #: underlying consumer driver is stopped.
    consumer_stopped_errors: ClassVar[Tuple[Type[BaseException], ...]] = ()

    # Mapping of TP to list of gap in offsets.
    _gap: MutableMapping[TP, List[int]]

    # Mapping of TP to list of acked offsets.
    _acked: MutableMapping[TP, List[int]]

    #: Fast lookup to see if tp+offset was acked.
    _acked_index: MutableMapping[TP, Set[int]]

    #: Keeps track of the currently read offset in each TP
    _read_offset: MutableMapping[TP, Optional[int]]

    #: Keeps track of the currently committed offset in each TP.
    _committed_offset: MutableMapping[TP, Optional[int]]

    #: The consumer.wait_empty() method will set this to be notified
    #: when something acks a message.
    _waiting_for_ack: Optional[asyncio.Future] = None

    #: Used by .commit to ensure only one thread is comitting at a time.
    #: Other thread starting to commit while a commit is already active,
    #: will wait for the original request to finish, and do nothing.
    _commit_fut: Optional[asyncio.Future] = None

    #: Set of unacked messages: that is messages that we started processing
    #: and that we MUST attempt to complete processing of, before
    #: shutting down or resuming a rebalance.
    _unacked_messages: MutableSet[Message]

    #: Time of when the consumer was started.
    _time_start: float

    # How often to poll and track log end offsets.
    _end_offset_monitor_interval: float

    _commit_every: Optional[int]
    _n_acked: int = 0

    _active_partitions: Optional[Set[TP]]
    _paused_partitions: Set[TP]
    _buffered_partitions: Set[TP]

    flow_active: bool = True
    can_resume_flow: Event

    def __init__(self,
                 transport: TransportT,
                 callback: ConsumerCallback,
                 on_partitions_revoked: PartitionsRevokedCallback,
                 on_partitions_assigned: PartitionsAssignedCallback,
                 *,
                 commit_interval: float = None,
                 commit_livelock_soft_timeout: float = None,
                 loop: asyncio.AbstractEventLoop = None,
                 **kwargs: Any) -> None:
        assert callback is not None
        self.transport = transport
        self.app = self.transport.app
        self.in_transaction = self.app.in_transaction
        self.callback = callback
        self._on_message_in = self.app.sensors.on_message_in
        self._on_partitions_revoked = on_partitions_revoked
        self._on_partitions_assigned = on_partitions_assigned
        self._commit_every = self.app.conf.broker_commit_every
        self.scheduler = self.app.conf.ConsumerScheduler()  # type: ignore
        self.commit_interval = (
            commit_interval or self.app.conf.broker_commit_interval)
        self.commit_livelock_soft_timeout = (
            commit_livelock_soft_timeout or
            self.app.conf.broker_commit_livelock_soft_timeout)
        self._gap = defaultdict(list)
        self._acked = defaultdict(list)
        self._acked_index = defaultdict(set)
        self._read_offset = defaultdict(lambda: None)
        self._committed_offset = defaultdict(lambda: None)
        self._unacked_messages = WeakSet()
        self._buffered_partitions = set()
        self._waiting_for_ack = None
        self._time_start = monotonic()
        self._end_offset_monitor_interval = self.commit_interval * 2
        self.randomly_assigned_topics = set()
        self.can_resume_flow = Event()
        self._reset_state()
        super().__init__(loop=loop or self.transport.loop, **kwargs)
        self.transactions = self.transport.create_transaction_manager(
            consumer=self,
            producer=self.app.producer,
            beacon=self.beacon,
            loop=self.loop,
        )

    def on_init_dependencies(self) -> Iterable[ServiceT]:
        """Return list of services this consumer depends on."""
        # We start the TransactionManager only if
        # processing_guarantee='exactly_once'
        if self.in_transaction:
            return [self.transactions]
        return []

    def _reset_state(self) -> None:
        self._active_partitions = None
        self._paused_partitions = set()
        self._buffered_partitions = set()
        self.can_resume_flow.clear()
        self.flow_active = True
        self._time_start = monotonic()

    async def on_restart(self) -> None:
        """Call when the consumer is restarted."""
        self._reset_state()
        self.on_init()

    def _get_active_partitions(self) -> Set[TP]:
        tps = self._active_partitions
        if tps is None:
            return self._set_active_tps(self.assignment())
        assert all(isinstance(x, TP) for x in tps)
        return tps

    def _set_active_tps(self, tps: Set[TP]) -> Set[TP]:
        xtps = self._active_partitions = ensure_TPset(tps)  # copy
        xtps.difference_update(self._paused_partitions)
        return xtps

    def on_buffer_full(self, tp: TP) -> None:
        active_partitions = self._get_active_partitions()
        active_partitions.discard(tp)
        self._buffered_partitions.add(tp)

    def on_buffer_drop(self, tp: TP) -> None:
        buffered_partitions = self._buffered_partitions
        if tp in buffered_partitions:
            active_partitions = self._get_active_partitions()
            active_partitions.add(tp)
            buffered_partitions.discard(tp)

    @abc.abstractmethod
    async def _commit(
            self,
            offsets: Mapping[TP, int]) -> bool:  # pragma: no cover
        ...

    async def perform_seek(self) -> None:
        """Seek all partitions to their current committed position."""
        read_offset = self._read_offset
        _committed_offsets = await self.seek_to_committed()
        read_offset.update({
            tp: offset if offset is not None and offset >= 0 else None
            for tp, offset in _committed_offsets.items()
        })
        committed_offsets = {
            ensure_TP(tp): offset if offset else None
            for tp, offset in _committed_offsets.items()
            if offset is not None
        }
        self._committed_offset.update(committed_offsets)

    @abc.abstractmethod
    async def seek_to_committed(self) -> Mapping[TP, int]:
        """Seek all partitions to their committed offsets."""
        ...

    async def seek(self, partition: TP, offset: int) -> None:
        """Seek partition to specific offset."""
        self.log.dev('SEEK %r -> %r', partition, offset)
        # reset livelock detection
        await self._seek(partition, offset)
        # set new read offset so we will reread messages
        self._read_offset[ensure_TP(partition)] = offset if offset else None

    @abc.abstractmethod
    async def _seek(self, partition: TP, offset: int) -> None:
        ...

    def stop_flow(self) -> None:
        """Block consumer from processing any more messages."""
        self.flow_active = False
        self.can_resume_flow.clear()

    def resume_flow(self) -> None:
        """Allow consumer to process messages."""
        self.flow_active = True
        self.can_resume_flow.set()

    def pause_partitions(self, tps: Iterable[TP]) -> None:
        """Pause fetching from partitions."""
        tpset = ensure_TPset(tps)
        self._get_active_partitions().difference_update(tpset)
        self._paused_partitions.update(tpset)

    def resume_partitions(self, tps: Iterable[TP]) -> None:
        """Resume fetching from partitions."""
        tpset = ensure_TPset(tps)
        self._get_active_partitions().update(tps)
        self._paused_partitions.difference_update(tpset)

    @abc.abstractmethod
    def _new_topicpartition(
            self, topic: str, partition: int) -> TP:  # pragma: no cover
        ...

    def _is_changelog_tp(self, tp: TP) -> bool:
        return tp.topic in self.app.tables.changelog_topics

    @Service.transitions_to(CONSUMER_PARTITIONS_REVOKED)
    async def on_partitions_revoked(self, revoked: Set[TP]) -> None:
        """Call during rebalancing when partitions are being revoked."""
        # NOTE:
        # The ConsumerRebalanceListener is responsible for calling
        # app.on_rebalance_start(), and this must have happened
        # before we get to this point (see aiokafka implementation).
        span = self.app._start_span_from_rebalancing('on_partitions_revoked')
        T = traced_from_parent_span(span)
        with span:
            # see comment in on_partitions_assigned
            # remove revoked partitions from active + paused tps.
            if self._active_partitions is not None:
                self._active_partitions.difference_update(revoked)
            self._paused_partitions.difference_update(revoked)
            await T(self._on_partitions_revoked, partitions=revoked)(
                revoked)

    @Service.transitions_to(CONSUMER_PARTITIONS_ASSIGNED)
    async def on_partitions_assigned(self, assigned: Set[TP]) -> None:
        """Call during rebalancing when partitions are being assigned."""
        span = self.app._start_span_from_rebalancing('on_partitions_assigned')
        T = traced_from_parent_span(span)
        with span:
            # remove recently revoked tps from set of paused tps.
            self._paused_partitions.intersection_update(assigned)
            # cache set of assigned partitions
            self._set_active_tps(assigned)
            # start callback chain of assigned callbacks.
            #   need to copy set at this point, since we cannot have
            #   the callbacks mutate our active list.
            await T(self._on_partitions_assigned, partitions=assigned)(
                assigned)
        self.app.on_rebalance_return()

    @abc.abstractmethod
    async def _getmany(self,
                       active_partitions: Optional[Set[TP]],
                       timeout: float) -> RecordMap:
        ...

    async def getmany(self,
                      timeout: float) -> AsyncIterator[Tuple[TP, Message]]:
        """Fetch batch of messages from server."""
        # records' contain mapping from TP to list of messages.
        # if there are two agents, consuming from topics t1 and t2,
        # normal order of iteration would be to process each
        # tp in the dict:
        #    for tp. messages in records.items():
        #        for message in messages:
        #           yield tp, message
        #
        # The problem with this, is if we have prefetched 16k records
        # for one partition, the other partitions won't even start processing
        # before those 16k records are completed.
        #
        # So we try round-robin between the tps instead:
        #
        #    iterators: Dict[TP, Iterator] = {
        #        tp: iter(messages)
        #        for tp, messages in records.items()
        #    }
        #    while iterators:
        #        for tp, messages in iterators.items():
        #            yield tp, next(messages)
        #            # remove from iterators if empty.
        #
        # The problem with this implementation is that
        # the records mapping is ordered by TP, so records.keys()
        # will look like this:
        #
        #  TP(topic='bar', partition=0)
        #  TP(topic='bar', partition=1)
        #  TP(topic='bar', partition=2)
        #  TP(topic='bar', partition=3)
        #  TP(topic='foo', partition=0)
        #  TP(topic='foo', partition=1)
        #  TP(topic='foo', partition=2)
        #  TP(topic='foo', partition=3)
        #
        # If there are 100 partitions for each topic,
        # it will process 100 items in the first topic, then 100 items
        # in the other topic, but even worse if partition counts
        # vary greatly, t1 has 1000 partitions and t2
        # has 1 partition, then t2 will end up being starved most of the time.
        #
        # We solve this by going round-robin through each topic.
        records, active_partitions = await self._wait_next_records(timeout)
        if records is None or self.should_stop:
            return

        records_it = self.scheduler.iterate(records)
        to_message = self._to_message  # localize
        if self.flow_active:
            for tp, record in records_it:
                if not self.flow_active:
                    break
                if active_partitions is None or tp in active_partitions:
                    highwater_mark = self.highwater(tp)
                    self.app.monitor.track_tp_end_offset(tp, highwater_mark)
                    # convert timestamp to seconds from int milliseconds.
                    yield tp, to_message(tp, record)

    async def _wait_next_records(
            self, timeout: float) -> Tuple[Optional[RecordMap],
                                           Optional[Set[TP]]]:
        if not self.flow_active:
            await self.wait(self.can_resume_flow)
        # Implementation for the Fetcher service.

        is_client_only = self.app.client_only

        active_partitions: Optional[Set[TP]]
        if is_client_only:
            active_partitions = None
        else:
            active_partitions = self._get_active_partitions()

        records: RecordMap = {}
        if is_client_only or active_partitions:
            # Fetch records only if active partitions to avoid the risk of
            # fetching all partitions in the beginning when none of the
            # partitions is paused/resumed.
            records = await self._getmany(
                active_partitions=active_partitions,
                timeout=timeout,
            )
        else:
            # We should still release to the event loop
            await self.sleep(1)
        return records, active_partitions

    @abc.abstractmethod
    def _to_message(self, tp: TP, record: Any) -> ConsumerMessage:
        ...

    def track_message(self, message: Message) -> None:
        """Track message and mark it as pending ack."""
        # add to set of pending messages that must be acked for graceful
        # shutdown.  This is called by transport.Conductor,
        # before delivering messages to streams.
        self._unacked_messages.add(message)
        # call sensors
        self._on_message_in(message.tp, message.offset, message)

    def ack(self, message: Message) -> bool:
        """Mark message as being acknowledged by stream."""
        if not message.acked:
            message.acked = True
            tp = message.tp
            offset = message.offset
            if self.app.topics.acks_enabled_for(message.topic):
                committed = self._committed_offset[tp]
                try:
                    if committed is None or offset > committed:
                        acked_index = self._acked_index[tp]
                        if offset not in acked_index:
                            self._unacked_messages.discard(message)
                            acked_index.add(offset)
                            acked_for_tp = self._acked[tp]
                            acked_for_tp.append(offset)
                            self._n_acked += 1
                            return True
                finally:
                    notify(self._waiting_for_ack)
        return False

    async def _wait_for_ack(self, timeout: float) -> None:
        # arm future so that `ack()` can wake us up
        self._waiting_for_ack = asyncio.Future(loop=self.loop)
        try:
            # wait for `ack()` to wake us up
            await asyncio.wait_for(
                self._waiting_for_ack, loop=self.loop, timeout=1)
        except (asyncio.TimeoutError,
                asyncio.CancelledError):  # pragma: no cover
            pass
        finally:
            self._waiting_for_ack = None

    @Service.transitions_to(CONSUMER_WAIT_EMPTY)
    async def wait_empty(self) -> None:
        """Wait for all messages that started processing to be acked."""
        wait_count = 0
        T = traced_from_parent_span()
        while not self.should_stop and self._unacked_messages:
            wait_count += 1
            if not wait_count % 10:  # pragma: no cover
                remaining = [(m.refcount, m) for m in self._unacked_messages]
                self.log.warning('wait_empty: Waiting for tasks %r', remaining)
                self.log.info(
                    'Agent tracebacks:\n%s',
                    self.app.agents.human_tracebacks(),
                )
            self.log.dev('STILL WAITING FOR ALL STREAMS TO FINISH')
            self.log.dev('WAITING FOR %r EVENTS', len(self._unacked_messages))
            gc.collect()
            await T(self.commit)()
            if not self._unacked_messages:
                break
            await T(self._wait_for_ack)(timeout=1)
            self._clean_unacked_messages()

        self.log.dev('COMMITTING AGAIN AFTER STREAMS DONE')
        await T(self.commit_and_end_transactions)()

    def _clean_unacked_messages(self) -> None:
        # remove actually acked messages from weakset.
        self._unacked_messages -= {
            message for message in self._unacked_messages
            if message.acked
        }

    async def commit_and_end_transactions(self) -> None:
        """Commit all safe offsets and end transaction."""
        await self.commit(start_new_transaction=False)

    async def on_stop(self) -> None:
        """Call when consumer is stopping."""
        if self.app.conf.stream_wait_empty:
            await self.wait_empty()
        else:
            await self.commit_and_end_transactions()

    @Service.task
    async def _commit_handler(self) -> None:
        interval = self.commit_interval

        await self.sleep(interval)
        async for sleep_time in self.itertimer(interval, name='commit'):
            await self.commit()

    @Service.task
    async def _commit_livelock_detector(self) -> None:  # pragma: no cover
        interval: float = self.commit_interval * 2.5
        await self.sleep(interval)
        async for sleep_time in self.itertimer(interval, name='livelock'):
            if not self.app.rebalancing:
                await self.verify_all_partitions_active()

    async def verify_all_partitions_active(self) -> None:
        now = monotonic()
        for tp in self.assignment():
            await self.sleep(0)
            if not self.should_stop:
                self.verify_event_path(now, tp)

    def verify_event_path(self, now: float, tp: TP) -> None:
        ...

    def verify_recovery_event_path(self, now: float, tp: TP) -> None:
        ...

    async def commit(self, topics: TPorTopicSet = None,
                     start_new_transaction: bool = True) -> bool:
        """Maybe commit the offset for all or specific topics.

        Arguments:
            topics: Set containing topics and/or TopicPartitions to commit.
        """
        if self.app.client_only:
            # client only cannot commit as consumer does not have group_id
            return False
        if await self.maybe_wait_for_commit_to_finish():
            # original commit finished, return False as we did not commit
            return False

        self._commit_fut = asyncio.Future(loop=self.loop)
        try:
            return await self.force_commit(
                topics,
                start_new_transaction=start_new_transaction,
            )
        finally:
            # set commit_fut to None so that next call will commit.
            fut, self._commit_fut = self._commit_fut, None
            # notify followers that the commit is done.
            notify(fut)

    async def maybe_wait_for_commit_to_finish(self) -> bool:
        """Wait for any existing commit operation to finish."""
        # Only one coroutine allowed to commit at a time,
        # and other coroutines should wait for the original commit to finish
        # then do nothing.
        if self._commit_fut is not None:
            # something is already committing so wait for that future.
            try:
                await self._commit_fut
            except asyncio.CancelledError:
                # if future is cancelled we have to start new commit
                pass
            else:
                return True
        return False

    @Service.transitions_to(CONSUMER_COMMITTING)
    async def force_commit(self,
                           topics: TPorTopicSet = None,
                           start_new_transaction: bool = True) -> bool:
        """Force offset commit."""
        sensor_state = self.app.sensors.on_commit_initiated(self)

        # Go over the ack list in each topic/partition
        commit_tps = list(self._filter_tps_with_pending_acks(topics))
        did_commit = await self._commit_tps(
            commit_tps, start_new_transaction=start_new_transaction)

        self.app.sensors.on_commit_completed(self, sensor_state)
        return did_commit

    async def _commit_tps(self,
                          tps: Iterable[TP],
                          start_new_transaction: bool) -> bool:
        commit_offsets = self._filter_committable_offsets(tps)
        if commit_offsets:
            try:
                # send all messages attached to the new offset
                await self._handle_attached(commit_offsets)
            except ProducerSendError as exc:
                await self.crash(exc)
            else:
                return await self._commit_offsets(
                    commit_offsets,
                    start_new_transaction=start_new_transaction)
        return False

    def _filter_committable_offsets(self, tps: Iterable[TP]) -> Dict[TP, int]:
        commit_offsets = {}
        for tp in tps:
            # Find the latest offset we can commit in this tp
            offset = self._new_offset(tp)
            # check if we can commit to this offset
            if offset is not None and self._should_commit(tp, offset):
                commit_offsets[tp] = offset
        return commit_offsets

    async def _handle_attached(self, commit_offsets: Mapping[TP, int]) -> None:
        for tp, offset in commit_offsets.items():
            app = cast(_App, self.app)
            attachments = app._attachments
            producer = app.producer
            # Start publishing the messages and return a list of pending
            # futures.
            pending = await attachments.publish_for_tp_offset(tp, offset)
            # then we wait for either
            #  1) all the attached messages to be published, or
            #  2) the producer crashing
            #
            # If the producer crashes we will not be able to send any messages
            # and it only crashes when there's an irrecoverable error.
            #
            # If we cannot commit it means the events will be processed again,
            # so conforms to at-least-once semantics.
            if pending:
                await cast(Service, producer).wait_many(pending)

    async def _commit_offsets(self, offsets: Mapping[TP, int],
                              start_new_transaction: bool = True) -> bool:
        table = terminal.logtable(
            [(str(tp), str(offset))
             for tp, offset in offsets.items()],
            title='Commit Offsets',
            headers=['TP', 'Offset'],
        )
        self.log.dev('COMMITTING OFFSETS:\n%s', table)
        assignment = self.assignment()
        committable_offsets: Dict[TP, int] = {}
        revoked: Dict[TP, int] = {}
        for tp, offset in offsets.items():
            if tp in assignment:
                committable_offsets[tp] = offset
            else:
                revoked[tp] = offset
        if revoked:
            self.log.info(
                'Discarded commit for revoked partitions that '
                'will be eventually processed again: %r',
                revoked,
            )
        if not committable_offsets:
            return False
        with flight_recorder(self.log, timeout=300.0) as on_timeout:
            did_commit = False
            on_timeout.info('+consumer.commit()')
            if self.in_transaction:
                did_commit = await self.transactions.commit(
                    committable_offsets,
                    start_new_transaction=start_new_transaction,
                )
            else:
                did_commit = await self._commit(committable_offsets)
            on_timeout.info('-consumer.commit()')
            if did_commit:
                on_timeout.info('+tables.on_commit')
                self.app.tables.on_commit(committable_offsets)
                on_timeout.info('-tables.on_commit')
        self._committed_offset.update(committable_offsets)
        self.app.monitor.on_tp_commit(committable_offsets)
        return did_commit

    def _filter_tps_with_pending_acks(
            self, topics: TPorTopicSet = None) -> Iterator[TP]:
        return (tp for tp in self._acked
                if topics is None or tp in topics or tp.topic in topics)

    def _should_commit(self, tp: TP, offset: int) -> bool:
        committed = self._committed_offset[tp]
        return committed is None or bool(offset) and offset > committed

    def _new_offset(self, tp: TP) -> Optional[int]:
        # get the new offset for this tp, by going through
        # its list of acked messages.
        acked = self._acked[tp]

        # We iterate over it until we find a gap
        # then return the offset before that.
        # For example if acked[tp] is:
        #   1 2 3 4 5 6 7 8 9
        # the return value will be: 9
        # If acked[tp] is:
        #  34 35 36 40 41 42 43 44
        #          ^--- gap
        # the return value will be: 36
        if acked:
            max_offset = max(acked)
            gap_for_tp = self._gap[tp]
            if gap_for_tp:
                gap_index = next((i for i, x in enumerate(gap_for_tp)
                                  if x > max_offset), len(gap_for_tp))
                gaps = gap_for_tp[:gap_index]
                acked.extend(gaps)
                gap_for_tp[:gap_index] = []
            acked.sort()
            # Note: acked is always kept sorted.
            # find first list of consecutive numbers
            batch = next(consecutive_numbers(acked))
            # remove them from the list to clean up.
            acked[:len(batch) - 1] = []
            self._acked_index[tp].difference_update(batch)
            # return the highest commit offset
            return batch[-1]
        return None

    async def on_task_error(self, exc: BaseException) -> None:
        """Call when processing a message failed."""
        await self.commit()

    def _add_gap(self, tp: TP, offset_from: int, offset_to: int) -> None:
        committed = self._committed_offset[tp]
        gap_for_tp = self._gap[tp]
        for offset in range(offset_from, offset_to):
            if committed is None or offset > committed:
                gap_for_tp.append(offset)

    async def _drain_messages(
            self, fetcher: ServiceT) -> None:  # pragma: no cover
        # This is the background thread started by Fetcher, used to
        # constantly read messages using Consumer.getmany.
        # It takes Fetcher as argument, because we must be able to
        # stop it using `await Fetcher.stop()`.
        callback = self.callback
        getmany = self.getmany
        consumer_should_stop = cast(Service, self)._stopped.is_set
        fetcher_should_stop = cast(Service, fetcher)._stopped.is_set

        get_read_offset = self._read_offset.__getitem__
        set_read_offset = self._read_offset.__setitem__
        flag_consumer_fetching = CONSUMER_FETCHING
        set_flag = self.diag.set_flag
        unset_flag = self.diag.unset_flag
        commit_every = self._commit_every
        acks_enabled_for = self.app.topics.acks_enabled_for

        yield_every = 100
        num_since_yield = 0
        sleep = asyncio.sleep

        try:
            while not (consumer_should_stop() or fetcher_should_stop()):
                set_flag(flag_consumer_fetching)
                ait = cast(AsyncIterator, getmany(timeout=1.0))

                # Sleeping because sometimes getmany is called in a loop
                # never releasing to the event loop
                await self.sleep(0)
                if not self.should_stop:
                    async for tp, message in ait:
                        num_since_yield += 1
                        if num_since_yield > yield_every:
                            await sleep(0)
                            num_since_yield = 0

                        offset = message.offset
                        r_offset = get_read_offset(tp)
                        if r_offset is None or offset > r_offset:
                            gap = offset - (r_offset or 0)
                            # We have a gap in income messages
                            if gap > 1 and r_offset:
                                acks_enabled = acks_enabled_for(message.topic)
                                if acks_enabled:
                                    self._add_gap(tp, r_offset + 1, offset)
                            if commit_every is not None:
                                if self._n_acked >= commit_every:
                                    self._n_acked = 0
                                    await self.commit()
                            await callback(message)
                            set_read_offset(tp, offset)
                        else:
                            self.log.dev('DROPPED MESSAGE ROFF %r: k=%r v=%r',
                                         offset, message.key, message.value)
                    unset_flag(flag_consumer_fetching)

        except self.consumer_stopped_errors:
            if self.transport.app.should_stop:
                # we're already stopping so ignore
                self.log.info('Broker stopped consumer, shutting down...')
                return
            raise
        except asyncio.CancelledError:
            if self.transport.app.should_stop:
                # we're already stopping so ignore
                self.log.info('Consumer shutting down for user cancel.')
                return
            raise
        except Exception as exc:
            self.log.exception('Drain messages raised: %r', exc)
            raise
        finally:
            unset_flag(flag_consumer_fetching)

    def close(self) -> None:
        """Close consumer for graceful shutdown."""
        ...

    @property
    def unacked(self) -> Set[Message]:
        """Return the set of currently unacknowledged messages."""
        return cast(Set[Message], self._unacked_messages)

class ConsumerThread(QueueServiceThread):
    """Consumer running in a dedicated thread."""

    app: AppT
    consumer: 'ThreadDelegateConsumer'
    transport: TransportT

    def __init__(self, consumer: 'ThreadDelegateConsumer',
                 **kwargs: Any) -> None:
        self.consumer = consumer
        self.transport = self.consumer.transport
        self.app = self.transport.app
        super().__init__(**kwargs)

    @abc.abstractmethod
    async def subscribe(self, topics: Iterable[str]) -> None:
        """Reset subscription (requires rebalance)."""
        ...

    @abc.abstractmethod
    async def seek_to_committed(self) -> Mapping[TP, int]:
        """Seek all partitions to their committed offsets."""
        ...

    @abc.abstractmethod
    async def commit(self, tps: Mapping[TP, int]) -> bool:
        """Commit offsets in topic partitions."""
        ...

    @abc.abstractmethod
    async def position(self, tp: TP) -> Optional[int]:
        """Return the current offset for partition."""
        ...

    @abc.abstractmethod
    async def seek_to_beginning(self, *partitions: TP) -> None:
        """Seek to the earliest offsets available for partitions."""
        ...

    @abc.abstractmethod
    async def seek_wait(self, partitions: Mapping[TP, int]) -> None:
        """Seek partitions to specific offsets and wait."""
        ...

    @abc.abstractmethod
    def seek(self, partition: TP, offset: int) -> None:
        """Seek partition to specific offset."""
        ...

    @abc.abstractmethod
    def assignment(self) -> Set[TP]:
        """Return the current assignment."""
        ...

    @abc.abstractmethod
    def highwater(self, tp: TP) -> int:
        """Return the last available offset in partition."""
        ...

    @abc.abstractmethod
    def topic_partitions(self, topic: str) -> Optional[int]:
        """Return number of configured partitions for topic by name."""
        ...

    @abc.abstractmethod
    def close(self) -> None:
        ...

    @abc.abstractmethod
    async def earliest_offsets(self, *partitions: TP) -> Mapping[TP, int]:
        """Return the earliest available offset for list of partitions."""
        ...

    @abc.abstractmethod
    async def highwaters(self, *partitions: TP) -> Mapping[TP, int]:
        """Return the last available offset for list of partitions."""
        ...

    @abc.abstractmethod
    async def getmany(self,
                      active_partitions: Optional[Set[TP]],
                      timeout: float) -> RecordMap:
        """Fetch batch of messages from server."""
        ...

    @abc.abstractmethod
    async def create_topic(self,
                           topic: str,
                           partitions: int,
                           replication: int,
                           *,
                           config: Mapping[str, Any] = None,
                           timeout: Seconds = 30.0,
                           retention: Seconds = None,
                           compacting: bool = None,
                           deleting: bool = None,
                           ensure_created: bool = False) -> None:
        """Create/declare topic on server."""
        ...

    async def on_partitions_revoked(
            self, revoked: Set[TP]) -> None:
        """Call on rebalance when partitions are being revoked."""
        await self.consumer.threadsafe_partitions_revoked(
            self.thread_loop, revoked)

    async def on_partitions_assigned(
            self, assigned: Set[TP]) -> None:
        """Call on rebalance when partitions are being assigned."""
        await self.consumer.threadsafe_partitions_assigned(
            self.thread_loop, assigned)

    @abc.abstractmethod
    def key_partition(self,
                      topic: str,
                      key: Optional[bytes],
                      partition: int = None) -> Optional[int]:
        """Hash key to determine partition number."""
        ...

    def verify_recovery_event_path(self, now: float, tp: TP) -> None:
        ...

class ThreadDelegateConsumer(Consumer):

    _thread: ConsumerThread

    #: Main thread method queue.
    #: The consumer is running in a separate thread, and so we send
    #: requests to it via a queue.
    #: Sometimes the thread needs to call code owned by the main thread,
    #: such as App.on_partitions_revoked, and in that case the thread
    #: uses this method queue.
    _method_queue: MethodQueue

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)
        self._method_queue = MethodQueue(loop=self.loop, beacon=self.beacon)
        self.add_dependency(self._method_queue)
        self._thread = self._new_consumer_thread()
        self.add_dependency(self._thread)

    @abc.abstractmethod
    def _new_consumer_thread(self) -> ConsumerThread:
        ...

    async def threadsafe_partitions_revoked(
            self,
            receiver_loop: asyncio.AbstractEventLoop,
            revoked: Set[TP]) -> None:
        """Call rebalancing callback in a thread-safe manner."""
        promise = await self._method_queue.call(
            receiver_loop.create_future(),
            self.on_partitions_revoked,
            revoked,
        )
        # wait for main-thread to finish processing request
        await promise

    async def threadsafe_partitions_assigned(
            self,
            receiver_loop: asyncio.AbstractEventLoop,
            assigned: Set[TP]) -> None:
        """Call rebalancing callback in a thread-safe manner."""
        promise = await self._method_queue.call(
            receiver_loop.create_future(),
            self.on_partitions_assigned,
            assigned,
        )
        # wait for main-thread to finish processing request
        await promise

    async def _getmany(self,
                       active_partitions: Optional[Set[TP]],
                       timeout: float) -> RecordMap:
        return await self._thread.getmany(active_partitions, timeout)

    async def subscribe(self, topics: Iterable[str]) -> None:
        """Reset subscription (requires rebalance)."""
        await self._thread.subscribe(topics=topics)

    async def seek_to_committed(self) -> Mapping[TP, int]:
        """Seek all partitions to the committed offset."""
        return await self._thread.seek_to_committed()

    async def position(self, tp: TP) -> Optional[int]:
        """Return the current position for partition."""
        return await self._thread.position(tp)

    async def seek_wait(self, partitions: Mapping[TP, int]) -> None:
        """Seek partitions to specific offsets and wait."""
        return await self._thread.seek_wait(partitions)

    async def _seek(self, partition: TP, offset: int) -> None:
        self._thread.seek(partition, offset)

    def assignment(self) -> Set[TP]:
        """Return the current assignment."""
        return self._thread.assignment()

    def highwater(self, tp: TP) -> int:
        """Return the last available offset for specific partition."""
        return self._thread.highwater(tp)

    def topic_partitions(self, topic: str) -> Optional[int]:
        """Return the number of partitions configured for topic by name."""
        return self._thread.topic_partitions(topic)

    async def earliest_offsets(self, *partitions: TP) -> Mapping[TP, int]:
        """Return the earliest offsets for a list of partitions."""
        return await self._thread.earliest_offsets(*partitions)

    async def highwaters(self, *partitions: TP) -> Mapping[TP, int]:
        """Return the last offset for a list of partitions."""
        return await self._thread.highwaters(*partitions)

    async def _commit(self, offsets: Mapping[TP, int]) -> bool:
        return await self._thread.commit(offsets)

    def close(self) -> None:
        """Close consumer for graceful shutdown."""
        self._thread.close()

    def key_partition(self,
                      topic: str,
                      key: Optional[bytes],
                      partition: int = None) -> Optional[int]:
        """Hash key to determine partition number."""
        return self._thread.key_partition(topic, key, partition=partition)

    def verify_recovery_event_path(self, now: float, tp: TP) -> None:
        return self._thread.verify_recovery_event_path(now, tp)