import sys
import os
import re
from fractions import Fraction
from enum import Enum
from collections import namedtuple

Time = namedtuple('Time', ['upper', 'lower', 'hyphen'])

class Note:
    """Member variables

    acc: accidental
        None, -1 (flat), 1 (sharp), 0 (natural)
    name: 0-7 as in numbered musical notation
    octave: int
    duration: Fraction
    line: # dash or underline
        line = 0: duration = 1
        line < 0: duration = 2 ** line
        line > 0: dash = line
    dot: # dots
    tie: 2-tuple of bool
        tie[0] = True: tied with the previous Note
        tie[1] = True: tied with the next Note
    """

    possible_ends = {}

    def __init__(self, acc, name, octave=0, duration=Fraction(1), dashes=0, underlines=0, dots=0, tie=(False, False)):
        self.acc = acc
        self.name = name
        self.octave = octave
        self.duration = duration
        if dashes is None:
            self.lines = None
            self.dots = None
        else:
            if dashes > 0:
                self.lines = dashes
            else:
                self.lines = -underlines
            self.dots = dots
        self.tie = list(tie)

    def copy(self):
        note = Note(self.acc, self.name, self.octave, self.duration)
        note.lines = self.lines
        note.dots = self.dots
        note.tie = self.tie.copy()
        return note

    @classmethod
    def init_possible_ends(cls, n, m):
        """Each beat is divided into (2**n) intervals, and there are m beats.

        Every duration consists of main duration and dots duration.
        The main duration consists of (2**k) intervals.
        """
        if (n, m) in cls.possible_ends:
            return
        num = m << n
        ends = [set() for _ in range(num)]
        # k < n
        for k in range(n):
            for a in range(0, num, 1 << k):
                start = a
                end = a + (1 << k)
                if end > num:
                    break
                ends[start].add(end)
                if (a >> k) % 2 == 0 and k > 0 and end + (1 << (k - 1)) < num:
                    ends[start + (1 << (k - 1))].add(end + (1 << (k - 1)))
                if (a >> k) % 2 == 1:
                    for j in range(k - 1, -1, -1):
                        start -= (1 << j)
                        if start < 0:
                            break
                        ends[start].add(end)
                else:
                    for j in range(k - 1, -1, -1):
                        end += (1 << j)
                        if end > num:
                            break
                        ends[start].add(end)
        # k >= n
        for k in range(n, n + m.bit_length()):
            for a in range(0, num, 1 << n):
                start = a
                end = a + (1 << k)
                if end > num:
                    break
                ends[start].add(end)
                if k > 0 and end + (1 << (k - 1)) < num:
                    ends[start + (1 << (k - 1))].add(end + (1 << (k - 1)))
                for j in range(k - 1, -1, -1):
                    start -= (1 << j)
                    if start < 0:
                        break
                    ends[start].add(end)
                start = a
                for j in range(k - 1, -1, -1):
                    end += (1 << j)
                    if end > num:
                        break
                    ends[start].add(end)
        for i in range(num):
            ends[i] = sorted(ends[i])
        cls.possible_ends[(n, m)] = ends

    @classmethod
    def split_note(cls, time, start_beat, note, _debug=False):
        debug_log = []
        if not time.hyphen:
            raise ValueError('no need to split note for time {}'.format(time))
        if time.upper is None:
            raise ValueError('cannot split note for time ?/{}'.format(time.lower))
        note.lines, note.dots = None, None      # ignore lines and dots
        duration = note.duration
        p = time.hyphen.bit_length() - 3

        if time.lower == 4:
            unit = (1 << p)
            if time.upper == 2:
                n, m = p + 1, 1
            elif time.upper == 3:
                n, m = p, 3
            elif time.upper == 4:
                n, m = p + 2, 1
            else:
                raise ValueError('unknown time.upper {}'.format(time.upper))
            n_unit, m_unit = 0, time.upper
        else:
            unit = int(Fraction(3, 2) * (1 << p))
            n, m = p - 1, 3
            if time.upper == 6:
                n_unit, m_unit = 0, 2
            elif time.upper == 9:
                n_unit, m_unit = 0, 3
            elif time.upper == 12:
                n_unit, m_unit = 2, 1
            else:
                raise ValueError('unknown time.upper {}'.format(time.upper))

        cls.init_possible_ends(n, m)
        ends = cls.possible_ends[(n, m)]
        cls.init_possible_ends(n_unit, m_unit)
        ends_unit = cls.possible_ends[(n_unit, m_unit)]
        end = int((start_beat + duration) * (1 << p))
        subnotes = []
        beat = start_beat
        debug_log.append('note {}'.format(note))

        while beat < start_beat + duration:
            start = int(beat * (1 << p))
            subend = None
            # multiple of units
            if start % unit == 0:
                for e_rel in ends_unit[start // unit]:
                    e = e_rel * unit
                    if e > end:
                        break
                    if time.lower == 8 and e != start + unit:  # only a unit is allowed for <time> X/8
                        continue
                    subend = e
                    debug_log.append('      E {}'.format(Fraction(subend - start, 1 << p)))
            # contains 'dots'
            base = start - start % (1 << n)
            for e_rel in ends[start % (1 << n)]:
                e = base + e_rel
                if e > end:
                    break
                length = e - start
                if (time.lower == 4 and length >= unit * 2) or \
                   (time.lower == 8 and length > unit):
                    break
                debug_log.append('      e {}'.format(Fraction(length, 1 << p)))
                if subend is None or e > subend:
                    subend = e
            if subend is None:
                raise ValueError('cannot find ending point in ({}, {}]'.format(start, end))
            end_beat = Fraction(subend, 1 << p)
            subnote = note.copy()
            subnote.duration = end_beat - beat
            if not subnotes:                            # first
                if end_beat != start_beat + duration:   # not last
                    subnote.tie[1] = True
            else:
                subnote.tie[0] = True
            subnotes.append(subnote)
            debug_log.append('   > {}'.format(subnote))
            beat = end_beat

        for i, subnote in enumerate(subnotes):
            if note.name == 0:
                subnote.tie[0] = False
                subnote.tie[1] = False
            else:
                if i > 0:
                    subnote.tie[0] = True
                if i < len(subnotes) - 1:
                    subnote.tie[1] = True

        if _debug:
            with open('log/split_note.log', 'w') as f:
                f.write('\n'.join(debug_log))
        return subnotes

    def __repr__(self):
        acc_str = {-1: 'b', 0: '%', 1: '#', None: ' '}
        oct_str = {-2: ',,', -1: ',', 0: '', 1: "'", 2: '"'}
        return '{}{}{:2} {:4}'.format(
                acc_str[self.acc], self.name, oct_str[self.octave], str(self.duration), self.tie)

class NodeType(Enum):
    """Type of node."""

    NOTE = 0
    DASH = 1
    DOT = 2

class Node:
    def __init__(self, note):
        if isinstance(note, Note):
            self.type = NodeType.NOTE
            self.value = note
            self.lines = note.lines
            self.dots = note.dots
            # calculate self.lines and self.dots from note.duration
            if self.lines is None or self.dots is None:
                duration = note.duration
                if duration <= 0:
                    raise ValueError('duration <= 0 for note {}'.format(note))
                numerator, denominator = duration.numerator, duration.denominator
                n = denominator.bit_length() - 1
                if (1 << n) != denominator:
                    raise ValueError('duration.denominator is not a power of 2 for note {}'.format(note))
                one_groups = list(filter(None, '{:b}'.format(numerator).split('0')))
                if len(one_groups) != 1:
                    raise ValueError('duration {} cannot be represented as a single note'.format(note.duration))
                m = numerator.bit_length() - 1
                if duration % Fraction(1) == 0:
                    self.lines = int(duration) - 1
                    self.dots = 0
                else:
                    if m > n:   # not allowed in numbered musical notation (e.g., duration = 7/2)
                        raise ValueError('duration {} is not integral, but too long for a note'.format(note.duration))
                    self.lines = m - n                  # self.lines <= 0
                    self.dots = len(one_groups[0]) - 1  # (number of dots) = (number of ones) - 1
        else:
            if note == '-':
                self.type = NodeType.DASH
            elif note == '.':
                self.type = NodeType.DOT
            else:
                raise ValueError('unknown node type for {}'.format(note))
            self.value = note
            self.lines = None
            self.dots = None
        self.text = None

    def __str__(self):
        if self.type == NodeType.NOTE:
            underlines = max(-self.lines, 0)
            if self.value.tie[1]:
                return '{} u{}  {} ~'.format(self.value, underlines, self.text)
            else:
                return '{} u{}  {}'.format(self.value, underlines, self.text)
        else:
            return ' {}'.format(self.value)

def parse_pitch(key, s):
    key_dict = dict(zip('ABCDEFG', [6, 7, 1, 2, 3, 4, 5]))
    extended_upper_name_dict = dict(zip('qwertyu', range(1, 8)))
    extended_lower_name_dict = dict(zip('zxcvbnm', range(1, 8)))
    acc_dict = {'': None, '#': 1, '$': -1, '%': 0}
    match = re.fullmatch(r"([#$%]?)([0-7a-zA-Z])([',]*)", s)
    if match is None:
        raise ValueError('wrong format for pitch {}'.format(s))
    acc, name, octave = match.groups()
    acc = acc_dict[acc]
    octave = octave.count("'") - octave.count(',')
    if name == '0':
        acc, name, octave = None, 0, 0
    elif key == 'solfa':
        if name in '1234567':
            name = int(name)
        elif name in extended_upper_name_dict:
            name = extended_upper_name_dict[name]
            octave += 1
        elif name in extended_lower_name_dict:
            name = extended_lower_name_dict[name]
            octave -= 1
        else:
            raise ValueError('{!r} is not allowed in <key> solfa'.format(s))
    else:
        if name in '1234567':
            name = int(name)
        elif name in 'cdefgabCDEFGAB':
            name = key_dict[name.upper()]
        else:
            raise ValueError('{!r} is not allowed in key {}'.format(name, key))
        if acc is not None:
            acc -= key[2][name]         # relative to key
        if key[0] <= 4:                 # <= #F major
            if name < key[0]:
                octave -= 1
        else:                           # >= G major
            if name >= key[0]:
                octave += 1
        name = (name - key[0]) % 7 + 1  # movable
    assert acc in [None, -1, 0, 1]
    assert name in [0, 1, 2, 3, 4, 5, 6, 7]
    assert octave in [-1, 0, 1]
    return acc, name, octave

class Song:
    """Member variables.

    key: (name, accidental, 8-tuple)
        tuple[i] indicates whether i is flat or sharp.
        For example, key = (2,  0, (?, 1,  0,  0,  1,  0,  0,  0)) stands for D major.
    melody: list of bars
        bar: (time, start_beat, [notes])
    lyrics: list of sections
        section: (tag, [strings])
    """
    def __init__(self):
        self.key = None
        self.melody = []
        self.lyrics = []

    def __str__(self):
        lines = [
            '<key> {}'.format(self.key),
            '<melody> {}'.format(self.melody),
            '<lyrics> {}'.format(self.lyrics),
        ]
        return '\n'.join(lines)

    def print(self):
        print('<key> {}'.format(self.key))
        for time, notes in self.melody:
            print('{}'.format('<time> {}/{}'.format(*time)))
            for note in notes:
                print('    {}'.format(note))
        for tag, lyrics in self.lyrics:
            print('<{}> {}'.format(tag, lyrics))

    def append_time_signature(self, time, s):
        """Append bars to self.melody.

        Split s into bars by '|'.
        If the duration of a bar exceeds time, split it further based on whether it is the first bar.
        """
        if not time:
            raise ValueError('unknown <time>')
        if not s:
            return

        pattern_pitch = r"[#$%]?[0-7a-zA-Z][',]*"
        pattern_pitches = r'\[(?:{})+\]'.format(pattern_pitch)
        pattern_duration = r'(?:[_=]+|-*)\.*(?:/3)?'
        pattern = '(~?)' + '({}|{})({})'.format(pattern_pitch, pattern_pitches, pattern_duration) + '(~?)'

        bars = s.split('|')
        for i, bar in enumerate(bars):
            if not bar:
                continue

            bar_duration = Fraction(0)
            note_list = []

            # parse notes in bar
            notes = re.findall(pattern, bar)
            if ''.join([''.join(note) for note in notes]) != bar:
                raise ValueError('wrong format for {} (notes = {})'.format(bar, notes))
            for tie0, pitches, duration, tie1 in notes:
                tie0, tie1 = (tie0 != ''), (tie1 != '')
                dots = duration.count('.')
                dashes = duration.count('-')
                underlines = duration.count('=') * 2 + duration.count('_')
                if dashes > 0 and underlines > 0:
                    raise ValueError('wrong format for {}'.format(notes))
                triplet = Fraction(1)
                if '/3' in duration:
                    triplet = Fraction(2, 3)
                if time.lower:
                    if pitches.startswith('[') and pitches.endswith(']'):
                        duration = (Fraction(dashes + 1, 1 << underlines)
                                    * (Fraction(2) - Fraction(1, 1 << dots)) * triplet)
                    else:
                        if dots or underlines or (triplet != 1):
                            raise ValueError('dots, underlines and triplets are not allowed'
                                             ' without brackets in <time> {}/{} hyphen={}'
                                             .format(*time))
                        duration = Fraction(dashes + 1, time.hyphen // 4)
                        dashes, underlines, dots = None, None, None
                else:
                    duration = (Fraction(dashes + 1, 1 << underlines)
                                * (Fraction(2) - Fraction(1, 1 << dots)) * triplet)
                pitches = pitches.lstrip('[').rstrip(']')
                pitches = re.findall('({})'.format(pattern_pitch), pitches)

                for k, pitch in enumerate(pitches):
                    acc, name, octave = parse_pitch(self.key, pitch)
                    tie = [tie0, tie1]
                    if k > 0:
                        tie[0] = False
                    if k < len(pitches) - 1:
                        tie[1] = False
                    # new Note
                    note = Note(acc, name, octave, duration, dashes, underlines, dots, tie)
                    note_list.append(note)
                    bar_duration += duration
            assert note_list

            # append to self.melody
            if time.upper is None:
                time_duration = bar_duration
            elif time.lower == 4:
                time_duration = Fraction(time.upper)
            else:
                time_duration = Fraction(time.upper, 2)
            if i == 0 and len(bars) > 1:        # first bar but not last
                beat = (time_duration - (bar_duration % time_duration)) % time_duration
            else:
                beat = Fraction(0)
            if beat > 0:
                self.melody.append((time, beat, []))            # new bar
            for note in note_list:
                assert note.duration > 0
                remaining_duration = note.duration
                first = True
                while remaining_duration > 0:
                    if beat == 0:
                        self.melody.append((time, beat, []))    # new bar
                    sub_duration = min(remaining_duration, time_duration - beat)
                    if not time.hyphen and sub_duration != remaining_duration:
                        raise ValueError('{} goes beyond one bar with time {}/{}'
                                         .format(note, time.upper, time.lower))
                    sub_note = note.copy()
                    sub_note.duration = sub_duration
                    if not first:
                        sub_note.tie[0] = True
                    self.melody[-1][-1].append(sub_note)
                    remaining_duration -= sub_duration
                    beat += sub_duration
                    assert beat <= time_duration
                    if beat == time_duration:
                        beat = 0
                    first = False

    def make_ties_consistent(self):
        """If note0 and note1 are consecutive, make sure that (note0.tie[1] == note1.tie[0])."""
        if not self.melody:
            return
        prev_tie = False
        for time, start_beat, notes in self.melody:
            if not notes:
                raise ValueError('empty bar in self.melody')
            for note in notes:
                if prev_tie:
                    note.tie[0] = True
                if note.name == 0:
                    note.tie = [False, False]
                prev_tie = note.tie[1]
        self.melody[0][-1][0].tie[0] = False    # first note
        self.melody[-1][-1][-1].tie[1] = False  # last note

    def try_split_notes(self):
        """Try to split note, and modify self.melody in place.

        If note.lines or note.dots is None, splitting is necessary.
        For these sub-notes, perform a 2nd-staged grouping.
        """
        for time, start_beat, notes in self.melody:
            if not time.hyphen:
                continue
            subnotes = []
            beat = start_beat
            for note in notes:
                if note.lines is None or note.dots is None:
                    subnotes += Note.split_note(time, beat, note)
                else:
                    subnotes.append(note)
                beat += note.duration
            notes[:] = subnotes

    def merge_melody_lyrics(self, _debug=False):
        """Return a list of sections.

        section: (tag, lines)
        line: [nodes, list of bars, list of ties, list of slurs]
        bar: (time, start_beat, node indices)
        tie: (node_idx1, node_idx2)
        """
        debug_log = []

        # calculate split indices according to self.lyrics
        sum_len = 0
        split_sections = {}
        split_lines = []
        for i, (tag, lyrics) in enumerate(self.lyrics):
            split_sections[sum_len] = tag
            for j, s in enumerate(lyrics):
                if s.startswith('~'):
                    raise ValueError("a line of lyrics cannot start with '~' ({})".format(s))
                split_lines.append(sum_len)
                sum_len += len(s)
        all_lyrics = ''.join([''.join(section[1]) for section in self.lyrics])
        num_words = len(all_lyrics)     # contains '~'

        # calculate the starting and ending index of slurs
        slur_lyrics_idx = {}
        prev_idx = None
        for idx, text in enumerate(all_lyrics):
            if text != '~':
                if prev_idx is not None and idx != prev_idx + 1:
                    slur_lyrics_idx[prev_idx] = idx - 1
                prev_idx = idx
        if prev_idx is not None:
            slur_lyrics_idx[prev_idx] = num_words - 1

        # split melody
        lyrics_idx = 0
        section_added, line_added = False, False    # whether these are added for this lyrics_idx
        line_node_idx_prev = -1
        sections = []
        for time, start_beat, notes in self.melody:
            if time.upper is None:
                time_duration = None
            elif time.lower == 4:
                time_duration = Fraction(time.upper)
            else:
                time_duration = Fraction(time.upper, 2)
            beat = start_beat
            for k, note in enumerate(notes):
                # new section
                if not note.tie[0] and not section_added and lyrics_idx in split_sections:
                    tag = split_sections[lyrics_idx]
                    sections.append((tag, []))
                    section_added = True
                # new line
                if not note.tie[0] and not line_added and lyrics_idx in split_lines:
                    if sections[-1][1] and note.name == 0 and (beat + note.duration) % time_duration == 0:
                        pass
                    else:
                        sections[-1][1].append([[], [], [], []])    # (nodes, bars, ties, slurs)
                        line_added = True
                        line_node_idx_prev = -1
                line = sections[-1][1][-1]
                nodes, bars, ties, slurs = line
                # new bar
                if k == 0 or not bars:
                    bars.append((time, beat, []))
                # append note Node
                node = Node(note)
                line_node_idx = len(nodes)
                bars[-1][-1].append(line_node_idx)
                if note.tie[0]:
                    ties.append((line_node_idx_prev, line_node_idx))
                    node.value.tie[0] = True
                    nodes[line_node_idx_prev].value.tie[1] = True
                else:
                    if note.name:
                        if lyrics_idx >= num_words:
                            raise ValueError('#notes > {} words'.format(num_words))
                        if all_lyrics[lyrics_idx] != '~':
                            node.text = all_lyrics[lyrics_idx]
                        lyrics_idx += 1
                        section_added, line_added = False, False
                nodes.append(node)
                line_node_idx_prev = line_node_idx
                # append dash Node's
                for _ in range(node.lines):
                    bars[-1][-1].append(len(nodes))
                    nodes.append(Node('-'))
                # append dot Node's
                for _ in range(node.dots):
                    bars[-1][-1].append(len(nodes))
                    nodes.append(Node('.'))
                beat += note.duration
        if lyrics_idx != num_words:
            raise ValueError('{} notes != {} words'.format(lyrics_idx, num_words))

        if _debug:
            with open('log/merge.log', 'w') as f:
                f.write('\n'.join(debug_log))

        return sections

    @classmethod
    def group_underlines(cls, sections):
        """Group underlines shared by contiguous notes, and find triplets by modifying sections in place.

        section: (tag, lines)
        line: [nodes, bars, ties, slurs, underlines_list, triplets]
        bar: (time, start_beat, node indices)
        tie: (node_idx1, node_idx2)
        underlines_list: list of underlines
            underlines[k]: list of underlines in depth k
            underline: [node_idx1, node_idx2]
        triplet: [node_idx1, node_idx2, node_idx3]
        """
        for tag, lines in sections:
            for line in lines:
                nodes, bars, ties, slurs = line
                underlines_list = [None]
                triplets = []
                triplet_duration = None
                for time, start_beat, idx_list in bars:
                    beat = start_beat
                    assert beat >= 0
                    idx_prev = -9
                    for idx in idx_list:
                        node = nodes[idx]
                        note = node.value
                        if node.type != NodeType.NOTE:
                            continue
                        # triplet
                        if (time.lower == 4 and beat % Fraction(1) == 0) or \
                           (time.lower == 8 and beat % Fraction(3, 2) == 0):
                            new_group = True
                            triplet_duration = None
                        else:
                            new_group = False
                        if note.duration.denominator % 3 == 0:       # triplet
                            if new_group or not triplets or note.duration != triplet_duration:
                                if triplets and len(triplets[-1]) != 3:
                                    raise ValueError('triplet with less than 3 notes')
                                triplets.append([idx])
                            elif len(triplets[-1]) == 3:
                                triplets.append([idx])
                            else:
                                triplets[-1].append(idx)
                            triplet_duration = note.duration
                        # underline
                        if node.lines < 0:
                            depth = -node.lines        # number of underlines
                            for _ in range(depth - len(underlines_list) + 1):
                                underlines_list.append([])
                            for k in range(1, depth + 1):
                                if new_group or not underlines_list[k]:
                                    underlines_list[k].append([idx, idx])
                                elif underlines_list[k][-1][1] == idx_prev:
                                    underlines_list[k][-1][1] = idx
                                else:
                                    underlines_list[k].append([idx, idx])
                        beat += note.duration
                        idx_prev = idx
                line.append(underlines_list)
                line.append(triplets)

    def to_tex_tikzpicture(self, output_dir=''):
        """Write environment tikzpicture source code to file if provided."""
        slides_file = os.path.join(output_dir, 'slides.tex')
        slides_output = []
        line_file_format = os.path.join(output_dir, 'line-{}{:02d}.tex')
        page_count = 0

        sections = self.merge_melody_lyrics()
        self.group_underlines(sections)

        for i, (tag, lines) in enumerate(sections):
            # new section
            line_count = 0
            for j, (nodes, bars, ties, slurs, underlines_list, triplets) in enumerate(lines):
                # new page
                if line_count % 2 == 0:
                    page_count += 1
                    if page_count > 1:
                        slides_output.append('\n')
                    slides_output.append('%%%%% PAGE {} %%%%%'.format(page_count))
                    slides_output.append(r'\newpage')
                    slides_output.append('')
                    if j == 0:      # first page in section
                        slides_output.append('% <{}>'.format(tag))
                        slides_output.append(r'\begin{nmntag}')
                        slides_output.append(r'\textmd{$<$\hspace{-0pt}' + tag + r'\hspace{-0pt}$>$}')
                        slides_output.append(r'\end{nmntag}')
                    else:
                        slides_output.append(r'\begin{nmnblank}')
                        slides_output.append(r'\end{nmnblank}')

                # new line
                line_lyrics = ''
                line_output = []
                line_output.append(r'\begin{tikzpicture}')
                line_output.append(r"""\tikzstyle{every node}=[inner sep=0pt]
\tikzstyle{dot}=[circle,fill=white,inner sep=0pt,text width=1.5pt]
\tikzstyle{lyrics}=[node distance=15pt]
\tikzstyle{tie}=[line width=0.5pt,bend left=45,min distance=4pt,max distance=5pt]
\tikzstyle{underline}=[line width=0.5pt]
\tikzstyle{tie0}=[line width=0.5pt,out=50,in=180,max distance=20pt]
\tikzstyle{tie1}=[line width=0.5pt,out=130,in=0,max distance=20pt]""")
                line_output.append('\n\n% nodes')
                line_output.append(r'\node at (0pt, 12pt) {}; % for space adjustment')

                pos = 0
                first_text_idx = None
                for k, (time, start_beat, idx_list) in enumerate(bars):
                    # new bar
                    if k > 0:
                        pos -= 2.5
                        line_output.append(r'\node at ({}pt,0) {{|}};'.format(pos))
                        pos += 7.5
                    for idx in idx_list:
                        node = nodes[idx]
                        note = node.value
                        line_output.append('')
                        if node.type != NodeType.NOTE:
                            pos -= 2.5
                            if node.type == NodeType.DASH:
                                line_output.append(r'\node at ({}pt,-1pt) {{-}};'.format(pos))
                            elif node.type == NodeType.DOT:
                                line_output.append(r'\node[dot] at ({}pt,0) {{}};'.format(pos))
                            pos += 7.5
                            continue
                        # name
                        line_output.append(r'\node (a{}) at ({}pt,0) {{{}}};'.format(idx, pos, note.name))
                        # acc
                        acc_dict = {-1: 'flat', 0: 'natural', 1: 'sharp'}
                        if note.acc is not None:
                            line_output.append(r'\node at ($(a{}.north west)+(-1pt,0)$){{\tiny$\{}$}};'
                                               .format(idx, acc_dict[note.acc]))
                        # octave
                        if note.octave > 0:
                            line_output.append(r'\node[dot,above of=a{},node distance=6pt] {{}};'.format(idx))
                        elif note.octave < 0:
                            node_distance = 7
                            if node.lines <= -3:
                                node_distance = 10
                            elif node.lines == -2:
                                node_distance = 9
                            elif node.lines == -1:
                                node_distance = 8
                            line_output.append(r'\node[dot,below of=a{},node distance={}pt] {{}};'
                                               .format(idx, node_distance))
                        # text
                        height = -17
                        if node.text:
                            if node.text in '每悔':
                                height += 1
                            text = '{0}{1}{0}'.format('\phantom{|}', node.text)
                            if first_text_idx is None:
                                first_text_idx = idx
                            line_output.append(r'\node[lyrics] (t{0}) at ($(a{0})+(0,{2}pt)$) {{{1}}};'
                                               .format(idx, text, height))
                            line_lyrics += node.text
                        elif first_text_idx is None:
                            text = r'\phantom{{{}}}'.format('天')
                            line_output.append(r'\node[lyrics] (t{0}) at ($(a{0})+(0,{2}pt)$) {{{1}}};'
                                               .format(idx, text, height))
                            first_text_idx = idx
                        pos += 10

                # ties
                line_output.append('\n\n% ties')
                for idx0, idx1 in ties:
                    dis = 2
                    if nodes[idx0].value.octave >= 1:
                        dis = 5
                    line_output.append(r'\draw[tie] (a{}.north) ++(0,{}pt) coordinate (tmp) to (a{}.north |- tmp);'
                                       .format(idx0, dis, idx1))

                # underlines
                line_output.append('\n\n% underlines')
                for depth, underlines in enumerate(underlines_list):
                    if depth == 0:
                        continue
                    for idx0, idx1 in underlines:
                        line_output.append(r'\draw[underline] (a{}.south west) ++(0,-{}pt)'.format(idx0, depth * 1.5)
                                           + r' coordinate (tmp) to (a{}.south east |- tmp);'.format(idx1))

                # triplets
                line_output.append('\n\n% triplets')
                for triplet in triplets:
                    dis0, dis1 = 2, 9
                    if nodes[triplet[0]].value.octave >= 1 or nodes[triplet[2]].value.octave >= 1:
                        dis0 = 5
                    if nodes[triplet[1]].value.octave >= 1:
                        dis1 = 12
                    line_output.append(r'\node[above of=a{},node distance={}pt] (tri) {{\tiny{{3}}}};'
                                       .format(triplet[1], dis1))
                    line_output.append(r'\draw[tie0] (a{}.north) +(0,{}pt) to ($(tri.west)+(-1pt,0)$);'
                                       .format(triplet[0], dis0))
                    line_output.append(r'\draw[tie1] (a{}.north) +(0,{}pt) to ($(tri.east)+(+1pt,0)$);'
                                       .format(triplet[2], dis0))

                line_output.append('')
                line_output.append(r'\end{tikzpicture}')
                line_output.append('')
                assert line_output[0] == r'\begin{tikzpicture}' and pos > 0
                line_output[0] = line_output[0] + '[xscale={}]'.format(110 / pos)

                line_file = line_file_format.format(chr(ord('a') + i), j)
                with open(line_file, 'w', encoding='utf-8') as f:
                    f.write('\n'.join(line_output))
                line_count += 1

                slides_output.append('\n% {}'.format(line_lyrics))
                slides_output.append(r'\begin{nmnline}')
                slides_output.append(r'\input{{{}}}'.format(line_file.split('/')[-1]))
                slides_output.append(r'\end{nmnline}')

        with open(slides_file, 'w', encoding='utf8') as f:
            f.write('\n'.join(slides_output))

    @classmethod
    def print(cls, sections):
        for tag, lines in sections:
            print('{:=^80}'.format(' ' + tag + ' '))
            for nodes, bars, ties, slurs, underlines_list, triplets in lines:
                print('-' * 50)
                for time, start_beat, a in bars:
                    print('<time> {}/{} beat={}'.format(time.upper, time.lower, start_beat))
                    for idx in a:
                        print('    <node {:02d}> {}'.format(idx, nodes[idx]))
                print('<ties> {}'.format(ties))
                print('<underlines>')
                for k, underlines in enumerate(underlines_list):
                    if k >= 1:
                        print('    depth {}: {}'.format(k, underlines))
                print('<triplets> {}'.format(triplets))

def parse_key(s):
    """Convert key string to ([1-7], 0 or 1 or -1, list of sharps or flats).

    s = [#$]?[a-gA-G]|[0-7][#$]?
    For example,
    s = '$D': D flat major
    s = '3#': A major
    key = (2,  0, (?, 1,  0,  0,  1,  0,  0,  0)): D major
    key = (6, -1, (?, 0, -1, -1,  0,  0, -1, -1)): A flat major
    """
    # construct table
    flat = [0, 7, 3, 6, 2, 5, 1, 4]
    sharp = [0, 4, 1, 5, 2, 6, 3, 7]
    sym2key = {}
    key2sym = {}
    for x in [-1, 1]:       # flat or sharp
        a = [0] * 8
        for n in range(8):  # number of x
            key = (4 * x * n) % 7 + 1
            tmp = 0
            if x == -1 and n >= 2:
                tmp = -1
            elif x == 1 and n >= 6:
                tmp = 1
            if x == -1:
                a[flat[n]] = -1
            else:
                a[sharp[n]] = 1
            sym2key[(n, x)] = (key, tmp, tuple(a))
            key2sym[(key, tmp)] = (n, x, tuple(a))

    # parse
    if not s:
        raise ValueError('empty key')
    elif s == 'solfa':
        return s
    if re.search(r'\d', s):     # s contains digits
        if len(s) == 1:
            if s[0] != '0':
                raise ValueError('wrong format for <key> {}'.format(s))
            return 1, 0
        if len(s) != 2 or s[0] not in '01234567' or s[1] not in '#$':
            raise ValueError('wrong format for <key> {}'.format(s))
        if s[1] == '#':
            x = 1
        else:
            x = -1
        return sym2key[(int(s[0]), x)]
    else:
        key = s[-1].upper()
        key_dict = dict(zip('ABCDEFG', [6, 7, 1, 2, 3, 4, 5]))
        if key not in key_dict:
            raise ValueError('wrong format for <key> {}'.format(s))
        key = key_dict[key]
        tmp = 0
        if len(s) == 2:
            if s[0] == '#':
                tmp = 1
            elif s[0] == '$':
                tmp = -1
            else:
                raise ValueError('wrong format for <key> {}'.format(s))
        return key, tmp, key2sym[(key, tmp)][2]

def parse_time(s):
    """Convert time signature string to a pair of integers."""
    hyphen = 4
    ss = s.split(maxsplit=1)
    if len(ss) > 1:
        hyphen = ss[1].replace(' ', '')
        if not hyphen.startswith('hyphen='):
            raise ValueError('wrong format for <time> {}'.format(s))
        hyphen = hyphen[7:]
        if hyphen not in ['4', '8', '16']:
            raise ValueError('only hyphen=[4,8,16] is allowed')
        hyphen = int(hyphen)
    ss = ss[0].replace(' ', '').split('/')
    if not ss:
        raise ValueError('empty <time>')
    if len(ss) != 2:
        raise ValueError('wrong format for <time> {}'.format(s))
    if ss[0] == '?':
        a, b = None, int(ss[1])
    else:
        a, b = int(ss[0]), int(ss[1])
    if (a, b) not in [(2, 4), (3, 4), (4, 4), (6, 8), (9, 8), (12, 8), (None, 4), (None, 8)]:
        raise ValueError('unrecognizable <time> {}/{}'.format(a, b))
    if hyphen and hyphen < b:
        raise ValueError('hyphen must >= {} for <time> {}/{}'.format(b, a, b))
    return Time(a, b, hyphen)

def load_song(melody_file, lyrics_file=None):
    """Load numbered musical notation and lyrics, and return Song."""
    song = Song()

    # melody
    with open(melody_file) as f:
        time = None
        s = ''
        for line in f:
            line = line.strip()
            if line == 'break':
                break
            elif not line or line.startswith('//'):     # blank or comment
                continue
            elif line.startswith('<key>'):
                if song.key:
                    raise ValueError('only one <key> is allowed')
                song.key = parse_key(line[5:].strip())
            elif line.startswith('<time>'):
                if time:
                    song.append_time_signature(time, s)
                time = parse_time(line[6:].strip())
                s = ''
            else:
                s += line.replace(' ', '')
        song.append_time_signature(time, s)
    song.make_ties_consistent()
    song.try_split_notes()

    # lyrics
    with open(lyrics_file, encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            s = line
            for c in ' ,.!?' + '　。，、！？':
                s = s.replace(c, '')
            if s == 'break':
                break
            elif not s or s.startswith('//'):
                continue
            elif s.startswith('<tag>'):
                tag = s[5:]
                song.lyrics.append((tag, []))
            else:
                if not song.lyrics:
                    raise ValueError('no <tag> specified before {}'.format(line))
                song.lyrics[-1][1].append(s)

    return song


#!/usr/bin/env python
from urllib import request
import json
from pprint import pprint

def get_wea(citycode):
    url_ci = 'http://www.weather.com.cn/data/cityinfo/%s.html' % citycode
    url_sk = 'http://www.weather.com.cn/data/sk/%s.html' % citycode
    url_zs = 'http://www.weather.com.cn/data/zs/%s.html' % citycode

    urls = {'城市信息':url_ci,'城市天气实况':url_sk,'城市天气指数':url_zs}
    res = {}
    for url in urls:
        r = request.urlopen(urls[url])
        data = r.read()
        info = json.loads(data)
        res[url] = info

    return res

if __name__ == '__main__':
    citycode = '101020100'
    wea = get_wea(citycode)
    pprint(wea)


class Config:

    # prepocessing patterns
    pre_processing = [
        # delete scripts
        r'<(script).*?</\1>(?s)',
        # delete styles
        r'<(style).*?</\1>(?s)',
        # delete meta
        r'<(meta).*?>',
        # delete ul's (optional)
        '<(ul).*?</\1>(?s)',
        # delete navigation
        r'<(nav).*?</\1>(?s)',
        # delete footer
        r'<(footer).*?</\1>(?s)',
        # delete header
        r'<(header).*?</\1>(?s)',
        # delete forms
        r'<(form).*?</\1>(?s)'
    ]

    # cut strategy AVG or CUSTOM
    STRATEGY = "CUSTOM"

    # custom cut coefficient
    CUSTOM_COEFF = 0.5

    # forced <p> finding
    FORCED = True

    # MAX_LENGTH to formatter
    MAX_LENGTH = 80


import pandas as pd
from collections import Counter, OrderedDict

df = pd.read_csv('faculty.csv')

# rename columns that have extra space in them
df.rename(columns={' degree': 'degree', ' title': 'title', ' email': 'email'}, inplace=True)

# ========================================================================================================

def clean_and_update_dataframe(df):
	'''
	Aim of function is to clean up dataframe columns even further. This requires one to:
	1. Make faculty degrees follow the same format and have no numeric values
	2. Generate new fields regarding seperating out the names and having a reformated one
	3. Have an abbreviated title, that is excluding the Biostatistics department from title
 	'''
	'''
	# this section of code is not was an inefficient way of updating degree column
	for index,element in enumerate(df['degree']):
		df['degree'][index] = element.strip().replace('.', '')
	'''
	df['degree'] = df['degree'].apply(lambda x: pd.Series(x.strip().replace('.', '') if x.isdigit() == False else ''))

	#seperate name into first and last name
	df['lastName'] = df['name'].apply(lambda x: pd.Series(x.split(' ')[-1]))
	df['firstName'] = df['name'].apply(lambda x: pd.Series(x.split(' ')[0]))
	df['middleName'] = df['name'].apply(lambda x: pd.Series(x.split(' ')[1] if len(x.split(' ')) == 3 else ' '))

	df['firstLastName'] = df['firstName'] + ', ' + df['lastName']
	df['abbrTitle'] = df['title'].apply(lambda x: pd.Series(x[:x.find('Professor')+len('Professor')]))

	return df

def dict_by_last_name(dFrame):
	df_updated = clean_and_update_dataframe(df)
	return {k: list(zip(g['degree'], g['abbrTitle'], g['email'])) for k,g in df_updated.groupby('lastName')}

print('--------------Dictionary by Faculty last name -----------------')
d1 = dict_by_last_name(df)
for key, val in d1.items():
	print(key, ': ', val)
print('\n')

# ========================================================================================================

def dict_by_last_and_first_name(dFrame):
	df_updated = clean_and_update_dataframe(df)

	# get relevant details, make firstLastName the index, transpose df and finally convert it to dictionary
	return df_updated[['firstLastName','degree','abbrTitle','email']].set_index('firstLastName').T.to_dict('list')

print('--------------Dictionary by Faculty (last, first) name -----------------')
d2 = dict_by_last_and_first_name(df)
for key, val in d2.items():
	print(key, ': ', val)
print('\n')

# ========================================================================================================

def sort_dict_by_last_name(converted_dict):
	return OrderedDict(sorted(converted_dict.items(), key=lambda t: t[0].split(',')[1].strip()))

print('--------------Sorted (by last name) Dictionary by Faculty (last, first) name -----------------')
d3 = sort_dict_by_last_name(d2)
for key, val in d3.items():
	print(key, ': ', val)


# vim: set fileencoding=utf-8 :
#
# Copyright (c) 2013 Daniel Truemper <truemped at googlemail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
from __future__ import (absolute_import, division, print_function,
                        with_statement)

import logging
import redis

from bbhack.Algorithms import Algorithms
from bbhack.base import BaseListener
from bbhack.updateStreamdrill import StreamDrillUpdater

LOG = logging.getLogger(__name__)

redis = redis.StrictRedis(host='localhost', port=6379, db=0)

class HashTagLogger(BaseListener):
    def __init__(self, zmq_sub_string, channel):
        super(HashTagLogger, self).__init__(zmq_sub_string, channel)
        self.algorithm = Algorithms()
        self.streamDrillUpdater = StreamDrillUpdater()

    def on_msg(self, tweet):

        if tweet['lang'] == 'en':
            if 'entities' in tweet and 'hashtags' in tweet['entities']:
                tags = tweet['entities']['hashtags']
                for tag in tags:
                    elem = tag['text']
                    # self.algorithm.computeHeavyHitter(elem)
                    self.algorithm.computeSketch(elem)
                    print(elem, self.algorithm.getSketchFor(elem))
                    redis.publish('hashtag_count', elem)
                    self.streamDrillUpdater.update(elem)
        return

def main():
    """Start the HashTagLogger."""

    import argparse

    p = argparse.ArgumentParser()
    p.add_argument('--zmq_sub_string', default='tcp://*:5556')
    p.add_argument('--channel', default='tweet.stream')

    options = p.parse_args()

    stream = HashTagLogger(options.zmq_sub_string, options.channel)
    # this call will block
    stream.start()


import heapq
class PriorityQueue:
    def __init__(self):
        self.elements = []

    def empty(self):
        return len(self.elements) == 0
    def put(self, item, priority):
        heapq.heappush(self.elements, (priority, item))

    def get(self):
        return heapq.heappop(self.elements)[1]

class searchAgent:
    def __init__(self, graph, nodes,costs, node, goal, type):
        self.nodes = nodes
        self.node = node
        self.goal = goal
        self.graph = graph
        self.nodes = nodes
        self.type = type
        self.costs = costs

    def cost(self, a,b):
        return self.costs[self.nodes[a]] + self.costs[self.nodes[b]]

    def heuristic(self,a,b,type):
        heuristic_type = {
            'manhattan' : self.manhattan_distance(a,b),
            'diagnol' : self.diagnol_distance(a,b),
            'simple' : self.simple_distance(a,b)
        }
        return heuristic_type[type]

    def manhattan_distance(self,a,b):
        (x1, y1) = a
        (x2, y2) = b
        return (self.cost(a,b) * (abs(x2 - x1) + abs(y1-y2)))

    def diagnol_distance(self,a,b):
        (x1, y1) = a
        (x2, y2) = b
        return (abs(x1+x2))**2 + (abs(y1+y2))**2

    def simple_distance(self,a,b):
        (x1, y1) = a
        (x2, y2) = b
        return abs(x1 - x2) + abs(y1 - y2)

    def a_star(self):
        frontier = PriorityQueue()
        frontier.put(self.node, 0)
        came_from = {}
        cost_so_far = {}
        came_from[self.node] = None
        cost_so_far[self.node] = 0
        while not frontier.empty():
            current = frontier.get()
            if current == self.goal:
                break
            neighbors  = self.graph[current]
            for neighbor in neighbors:
                new_cost = cost_so_far[current] + self.cost(current,neighbor)
                if neighbor not in cost_so_far or new_cost  < cost_so_far[neighbor]:
                    cost_so_far[neighbor] = new_cost
                    priority = new_cost + self.heuristic(neighbor,self.goal,self.type)
                    frontier.put(neighbor,priority)
                    came_from[neighbor] = current

        return came_from, cost_so_far

    #newmethod for PACMAN AI
    def dijktra_search(self):
        frontier = PriorityQueue()
        frontier.put(self.node,0)
        came_from = {}
        cost_so_far = {}
        came_from[self.node] = None
        cost_so_far[self.node] = 0
        while not frontier.empty():
            current = frontier.get()
            if current == self.goal:
                break
            neighbors = self.graph[current]
            for neighbor in neighbors:
                new_cost = cost_so_far[current] + self.cost(current,neighbor)
                if neighbor not in cost_so_far or new_cost < cost_so_far[neighbor]:
                    cost_so_far[neighbor] = new_cost
                    priority = new_cost
                    frontier.put(neighbor,priority)
                    came_from[neighbor] = current


import sys
import socket
import base64
import os

args = sys.argv

#args = ['', '']
#args[0] = "141.64.174.92"
#args[1] = "text.txt"

def sendMessage(message, protocol):
    '''Send Message with given protocol (str)'''
    s.send(str.encode(protocol + "\r\n"))
    s.send(str.encode("peer notify\r\n"))
    s.send(str.encode(sys.argv[1] + "\r\n"))
    s.send(str.encode(bytes.decode(message) + "\r\n"))
    s.send(str.encode("dslp/end\r\n"))

if len(args) < 3:
    print("Please set two command line arguments!")
else:

    adress = "dbl44.beuth-hochschule.de"
    port = 44444
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.connect((adress, port))
        print("Connect to server!")
    except:
        print("Can't connect to server!")
        s.close()
        SystemExit()

    file = args[2]
    if os.path.exists(file):

        file = open(file, 'rb')
        file = file.read()
        file_encode = base64.encodebytes(file)
        print(file_encode)

        sendMessage(file_encode, "dslp/1.2")
        print("Send file!")

        s.close()
        SystemExit()
    else:
        print("Path does not exist!")
s.close()
SystemExit()


import socket
from tkinter import*
import time
from my_socket import *
class Application:
    def __init__(self,master=None):
        self.fontePadrao = ("Arial", "10")
        self.PORT = 1234            # Porta que o Servidor esta

        self.server = serverSocket(5000)
        self.server.openServer()

        self.client = clientSocket(1234)
        time.sleep(0.5)
        self.client.connect()

        """Inicia o primeiro container"""
        self.primeiroContainer = Frame(master)
        self.primeiroContainer["pady"] = 5
        self.primeiroContainer.pack()

        self.titulo = Label(self.primeiroContainer, text="Aplicação Recebe")
        self.titulo["font"] = ("Arial", "10", "bold")
        self.titulo.pack()

        self.texto = Text(self.primeiroContainer)
        self.texto.insert(END, "Server conectado na porta {} \n".format(self.server.PORT))
        self.texto.insert(END, "Cliente conectado na porta {} \n".format(self.client.PORT))
        self.texto["font"] = ("Arial", "10", "bold")
        self.texto["state"] = DISABLED
        self.texto.pack()
        """Finaliza o primeiro container"""

        """Inicia o segundo container"""
        self.segundoContainer = Frame(master)
        self.segundoContainer["padx"] = 20
        self.segundoContainer.pack()

        self.arquivoLabel = Label(self.segundoContainer, font=self.fontePadrao)
        self.arquivoLabel.pack()
        """Finaliza o segundo container"""

        """Inicia o terceiro container"""
        self.terceiroContainer = Frame(master)
        self.terceiroContainer["padx"] = 20
        self.terceiroContainer.pack()

        self.campoTexto = Entry(self.terceiroContainer)
        self.campoTexto.insert(END,"")
        self.campoTexto["width"] = 30
        self.campoTexto["font"] = self.fontePadrao
        self.campoTexto.pack()

        """Finaliza o terceiro container"""

        """Inicia o quarto container"""
        self.quartoContainer = Frame(master)
        self.quartoContainer["pady"] = 10
        self.quartoContainer.pack()

        self.buttonGet = Button(self.quartoContainer,command=self.getText)
        self.buttonGet["text"] = "Recebe Mensagem"
        self.buttonGet["font"] = ("Calibri", "8")
        self.buttonGet["width"] = 12
        self.buttonGet.pack(side=LEFT)

        self.buttonSend = Button(self.quartoContainer,command=self.sendText)
        self.buttonSend["text"] = "Envia Mensagem"
        self.buttonSend["font"] = ("Calibri", "8")
        self.buttonSend["width"] = 12
        self.buttonSend.pack(side=RIGHT)
        """Finaliza o quarto container"""

    def getText(self):
        try:
            self.texto["state"] = NORMAL
            msg = self.server.getData()
            self.texto.insert(END,'Ele: ' + msg + '\n')
            self.texto["state"] = DISABLED

        except socket.timeout as e:
            print(e)

    def sendText(self):
        msg = self.campoTexto.get()
        self.campoTexto.delete(0,END)
        try:
            self.client.sendData(msg)
            print("enviou")
        except:
            pass
        self.texto["state"] = NORMAL
        self.texto.insert(END,"Eu: "+ msg+'\n')
        self.texto["state"] = DISABLED

root = Tk()
root.title("Receive")
Application = Application(root)
root.mainloop()


#!/usr/bin/env python
"""
Serial program that runs a queue of serial simulations.
Usage: ./serial_queue.py <executable> <input_directory>
"""
import os
import sys
import glob
import time
from subprocess import Popen

def input_files(my_path):
    return glob.glob(my_path+'/input*.txt')

def run(executable, inputs):
    for inp in inputs:
        with open(inp+'.out', 'w') as out_file:
           pid = Popen([executable, inp], stdout=out_file)
           pid.wait()

if __name__ == '__main__':
    executable = os.path.abspath(sys.argv[1])
    input_dir = os.path.abspath(sys.argv[2])

    inputs = input_files(input_dir)
    print("Number of elaborations: {:d}".format(len(inputs)))

    t1 = time.time()
    run(executable, inputs)
    t2 = time.time()

    print("Elapsed time {:5.2f}".format(t2-t1))


# uncompyle6 version 3.7.4
# Python bytecode 3.7 (3394)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.macosx-10.12-x86_64/egg/dicom_tools/make_histo.py
# Compiled at: 2018-09-14 09:20:48
# Size of source mod 2**32: 11709 bytes
from __future__ import print_function
try:
    import ROOT
    ROOTfound = True
except ImportError:
    ROOTfound = False
    print('WARNING make_histo.py: ROOT not found')

import numpy as np
from skimage.feature import greycomatrix, greycoprops
from skimage import exposure
from dicom_tools.rescale import rescale8bit
import dicom_tools.calculateMeanInROI as calculateMeanInROI
import dicom_tools.fractal as fractal
from scipy import ndimage
from dicom_tools.gaussianlaplace import GaussianLaplaceFilter

def make_histo(data, mask, suffix='', verbose=False, ROInorm=False, normalize=False, scala=False, filtered=False):
    nbin = 10000
    binmin = data.min() * 0.8
    binmax = data.max() * 1.2
    if scala:
        binmin = -5
        binmax = 5
    if normalize:
        layerOfMax = np.where(data == data.max())[0][0]
        normformaxbin = calculateMeanInROI(data[layerOfMax], ROInorm[layerOfMax], verbose)
        binmax = data.max() / normformaxbin
        binmin = 0
        nbin = int(data.max() - binmin)
        bindim = (binmax - binmin) / nbin
        if verbose:
            print('make_histo: layerOfMax', layerOfMax, 'data.max()', data.max(), 'binmax', binmax, 'nbin', nbin)
    if verbose:
        print('make_histo: bin min:', binmin, 'bin max:', binmax, 'nbin:', nbin)
    nFette = len(data)
    his = ROOT.TH1F('histo' + suffix, 'histo', nbin, binmin, binmax)
    hEntries = ROOT.TH1F('hEntries' + suffix, 'Entries', nFette, -0.5, nFette + 0.5)
    hMean = ROOT.TH1F('hMean' + suffix, 'Mean', nFette, -0.5, nFette + 0.5)
    hStdDev = ROOT.TH1F('hStdDev' + suffix, 'StdDev', nFette, -0.5, nFette + 0.5)
    hSkewness = ROOT.TH1F('hSkewness' + suffix, 'Skewness', nFette, -0.5, nFette + 0.5)
    hKurtosis = ROOT.TH1F('hKurtosis' + suffix, 'Kurtosis', nFette, -0.5, nFette + 0.5)
    hfra = ROOT.TH1F('hfra' + suffix, 'fra', nFette, -0.5, nFette + 0.5)
    hCfra = ROOT.TH1F('hCfra' + suffix, 'hCfra', nFette, -0.5, nFette + 0.5)
    hdissH = ROOT.TH1F('hdissH' + suffix, 'Horizontal dissimilarity', nFette, -0.5, nFette + 0.5)
    hcorrH = ROOT.TH1F('hcorrH' + suffix, 'Horizontal correlation', nFette, -0.5, nFette + 0.5)
    henerH = ROOT.TH1F('henerH' + suffix, 'Horizontal energy', nFette, -0.5, nFette + 0.5)
    hcontH = ROOT.TH1F('hcontH' + suffix, 'Horizontal contrast', nFette, -0.5, nFette + 0.5)
    hhomoH = ROOT.TH1F('hhomoH' + suffix, 'Horizontal homogeneity', nFette, -0.5, nFette + 0.5)
    hdissV = ROOT.TH1F('hdissV' + suffix, 'Vertical dissimilarity', nFette, -0.5, nFette + 0.5)
    hcorrV = ROOT.TH1F('hcorrV' + suffix, 'Vertical correlation', nFette, -0.5, nFette + 0.5)
    henerV = ROOT.TH1F('henerV' + suffix, 'Vertical energy', nFette, -0.5, nFette + 0.5)
    hcontV = ROOT.TH1F('hcontV' + suffix, 'Vertical contrast', nFette, -0.5, nFette + 0.5)
    hhomoV = ROOT.TH1F('hhomoV' + suffix, 'Vertical homogeneity', nFette, -0.5, nFette + 0.5)
    hdissPQ = ROOT.TH1F('hdissPQ' + suffix, '+45 degree dissimilarity', nFette, -0.5, nFette + 0.5)
    hcorrPQ = ROOT.TH1F('hcorrPQ' + suffix, '+45 degree correlation', nFette, -0.5, nFette + 0.5)
    henerPQ = ROOT.TH1F('henerPQ' + suffix, '+45 degree energy', nFette, -0.5, nFette + 0.5)
    hcontPQ = ROOT.TH1F('hcontPQ' + suffix, '+45 degree contrast', nFette, -0.5, nFette + 0.5)
    hhomoPQ = ROOT.TH1F('hhomoPQ' + suffix, '+45 degree homogeneity', nFette, -0.5, nFette + 0.5)
    hdissMQ = ROOT.TH1F('hdissMQ' + suffix, '-45 degree dissimilarity', nFette, -0.5, nFette + 0.5)
    hcorrMQ = ROOT.TH1F('hcorrMQ' + suffix, '-45 degree correlation', nFette, -0.5, nFette + 0.5)
    henerMQ = ROOT.TH1F('henerMQ' + suffix, '-45 degree energy', nFette, -0.5, nFette + 0.5)
    hcontMQ = ROOT.TH1F('hcontMQ' + suffix, '-45 degree contrast', nFette, -0.5, nFette + 0.5)
    hhomoMQ = ROOT.TH1F('hhomoMQ' + suffix, '-45 degree homogeneity', nFette, -0.5, nFette + 0.5)
    allhistos = []
    histogiafatti = []
    histogclm = []
    for layer in xrange(0, nFette):
        fetta = data[layer]
        if scala:
            media = np.mean(fetta)
            rms = np.std(fetta)
            fetta = (fetta - media) / rms
        fetta8bit = rescale8bit(fetta)
        fettaROI = mask[layer]
        glcmdata = mask[layer] * fetta8bit
        glcm1 = greycomatrix(glcmdata, [1], [0], 256, symmetric=True, normed=True)
        glcm2 = greycomatrix(glcmdata, [1], [np.pi / 2], 256, symmetric=True, normed=True)
        glcm3 = greycomatrix(glcmdata, [1], [np.radians(45)], 256, symmetric=True, normed=True)
        glcm4 = greycomatrix(glcmdata, [1], [np.radians(-45)], 256, symmetric=True, normed=True)
        hdissH.SetBinContent(layer, greycoprops(glcm1, 'dissimilarity')[(0, 0)])
        hcorrH.SetBinContent(layer, greycoprops(glcm1, 'correlation')[(0, 0)])
        henerH.SetBinContent(layer, greycoprops(glcm1, 'energy')[(0, 0)])
        hcontH.SetBinContent(layer, greycoprops(glcm1, 'contrast')[(0, 0)])
        hhomoH.SetBinContent(layer, greycoprops(glcm1, 'homogeneity')[(0, 0)])
        hdissV.SetBinContent(layer, greycoprops(glcm2, 'dissimilarity')[(0, 0)])
        hcorrV.SetBinContent(layer, greycoprops(glcm2, 'correlation')[(0, 0)])
        henerV.SetBinContent(layer, greycoprops(glcm2, 'energy')[(0, 0)])
        hcontV.SetBinContent(layer, greycoprops(glcm2, 'contrast')[(0, 0)])
        hhomoV.SetBinContent(layer, greycoprops(glcm2, 'homogeneity')[(0, 0)])
        hdissPQ.SetBinContent(layer, greycoprops(glcm3, 'dissimilarity')[(0, 0)])
        hcorrPQ.SetBinContent(layer, greycoprops(glcm3, 'correlation')[(0, 0)])
        henerPQ.SetBinContent(layer, greycoprops(glcm3, 'energy')[(0, 0)])
        hcontPQ.SetBinContent(layer, greycoprops(glcm3, 'contrast')[(0, 0)])
        hhomoPQ.SetBinContent(layer, greycoprops(glcm3, 'homogeneity')[(0, 0)])
        hdissMQ.SetBinContent(layer, greycoprops(glcm4, 'dissimilarity')[(0, 0)])
        hcorrMQ.SetBinContent(layer, greycoprops(glcm4, 'correlation')[(0, 0)])
        henerMQ.SetBinContent(layer, greycoprops(glcm4, 'energy')[(0, 0)])
        hcontMQ.SetBinContent(layer, greycoprops(glcm4, 'contrast')[(0, 0)])
        hhomoMQ.SetBinContent(layer, greycoprops(glcm4, 'homogeneity')[(0, 0)])
        thishisto = ROOT.TH1F('h' + str(layer) + suffix, 'h' + str(layer), nbin, binmin, binmax)
        meaninroi = 1
        frattale = 0
        frattalecont = 0
        if fettaROI.max() > 0:
            frattale = fractal().frattali(fettaROI)
            frattalecont = fractal().frattali(fetta * np.subtract(fettaROI, ndimage.binary_erosion(fettaROI).astype(fettaROI.dtype)))
        if fettaROI.any():
            if normalize:
                meaninroi = calculateMeanInROI(fetta, ROInorm[layer], verbose)
                if verbose:
                    print('make_histo: layer', layer, 'meaninroi', meaninroi)
            if fettaROI.any():
                if verbose:
                    print('non zero pixels:', np.count_nonzero(fetta))
            for val, inROI in zip(fetta.ravel(), fettaROI.ravel()):
                if inROI > 0:
                    if normalize:
                        val = val / meaninroi
                    if val > binmax:
                        print('make_histo: Warning in layer', layer, 'there is a value in overflow:', val, 'normalization', meaninroi)
                    if val > 0:
                        his.Fill(val)
                        thishisto.Fill(val)

            hEntries.SetBinContent(layer, thishisto.GetEntries())
            hMean.SetBinContent(layer, thishisto.GetMean())
            hStdDev.SetBinContent(layer, thishisto.GetStdDev())
            hSkewness.SetBinContent(layer, thishisto.GetSkewness())
            hKurtosis.SetBinContent(layer, thishisto.GetKurtosis())
            hfra.SetBinContent(layer, frattale)
            hCfra.SetBinContent(layer, frattalecont)
        if verbose:
            print('make_histo', 'layer', layer, 'entries', thishisto.GetEntries())
        allhistos.append(thishisto)

    histogiafatti.append(hEntries)
    histogiafatti.append(hMean)
    histogiafatti.append(hStdDev)
    histogiafatti.append(hSkewness)
    histogiafatti.append(hKurtosis)
    histogiafatti.append(hfra)
    histogiafatti.append(hCfra)
    histogclm.append(hdissH)
    histogclm.append(hcorrH)
    histogclm.append(henerH)
    histogclm.append(hcontH)
    histogclm.append(hhomoH)
    histogclm.append(hdissV)
    histogclm.append(hcorrV)
    histogclm.append(henerV)
    histogclm.append(hcontV)
    histogclm.append(hhomoV)
    histogclm.append(hdissPQ)
    histogclm.append(hcorrPQ)
    histogclm.append(henerPQ)
    histogclm.append(hcontPQ)
    histogclm.append(hhomoPQ)
    histogclm.append(hdissMQ)
    histogclm.append(hcorrMQ)
    histogclm.append(henerMQ)
    histogclm.append(hcontMQ)
    histogclm.append(hhomoMQ)
    return (
     his, allhistos, histogiafatti, histogclm)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 17 13:18:46 2020

@author: root
"""
from pathlib import Path
import pandas as pd
import wget
import os
from process_jsons import *
import shutil
import pathlib
import argparse
import sys

def copy_all_files(src, dst):
    src_files = os.listdir(src)
    for file_name in src_files:
        full_file_name = os.path.join(src, file_name)
        if os.path.isfile(full_file_name):
            shutil.copy(full_file_name, dst)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("mavd_path", type=str, default="/clusteruy/home/ihounie/MAVD")
    parser.add_argument("output_path", type=str, default="/clusteruy/home/ihounie/urban-sound-tagging-baseline/sonyc-mavd/data")
    parser.add_argument('-c','--copy', action='store_true', help="copy files on output dir")
    parser.add_argument('-e','--extract', action='store_true', help="generate jsons from mavd")
    parser.add_argument('-na','--no_audio', action='store_false', help="do not extract audio from MAVD")
    parser.add_argument('-d','--duration', type=float, default=10.0, help="audio segment duration in seconds")
    parser.add_argument('-ml','--min_label', type=float, default=3.0, help="minimum event length of considered labels")
    parser.add_argument('-hop','--hop_size', type=float, default=1.0, help="hop for extracting overlapping train segments")
    parser.add_argument('-tl','--tagless', type=float, default=2.0, help="increase to extract more tagless segments to improve class balance")

    args = parser.parse_args()
    mavd_path = args.mavd_path
    output_path = args.output_path

    if args.extract:
        print("extracting audios from mavd")
        pathlib.Path(os.path.join(mavd_path, "train")).mkdir(parents=True, exist_ok=True)
        pathlib.Path(os.path.join(mavd_path, "validate")).mkdir(parents=True, exist_ok=True)
        process_mavd(duration = args.duration, #seconds,
                     clip_length = args.duration,
                     min_label = args.min_label,
                     overlap  = args.hop_size,
                    mavd_path = mavd_path, split="train", train_frac = 0.5, tagless_frac = args.tagless)
        jsons2csv(mode="fine",mavd_path = mavd_path, split="train", split_out ="train", audio=args.no_audio)
        jsons2csv(mode="fine",mavd_path = mavd_path, split="train", split_out ="validate", audio=args.no_audio)
    annot_csv_pth_train = os.path.join(mavd_path, "train","annotations.csv")
    train =  pd.read_csv(annot_csv_pth_train)
    print("*"*20)
    print(f" {len(train.index)} train samples")
    print("*"*20)

    annot_csv_pth_val = os.path.join(mavd_path, "validate","annotations.csv")
    val =  pd.read_csv(annot_csv_pth_val)
    print("*"*20)
    print(f" {len(val.index)} test samples")
    print("*"*20)
    assert(len(list(set(train.columns) - set(val.columns)))==0)
    assert(len(list(set(val.columns) - set(train.columns)))==0)

    merged = pd.concat([train, val])
    print(f"Total samlples: {len(merged.index)}")

    Path(output_path).mkdir(parents=True, exist_ok=True)
    print(os.getcwd())
    merged.to_csv(os.path.join(output_path,'annotations.csv'), index=False)
    #############################################################
    # Another annotations file with test files as validation set
    ############################################################
    #test["split"] = "validate"
    #merged_test = pd.concat([train, test])
    #merged_test.to_csv(os.path.join(output_path,'annotations_test.csv'), index=False)

    #out_test_dir = os.path.join(output_path,"test")
    out_train_dir = os.path.join(output_path,"train")
    out_val_dir = os.path.join(output_path,"validate")

    #pathlib.Path(out_test_dir).mkdir(parents=True, exist_ok=True)
    pathlib.Path(out_train_dir).mkdir(parents=True, exist_ok=True)
    pathlib.Path(out_val_dir).mkdir(parents=True, exist_ok=True)
    if args.copy:
        print("Copying Validation files...")
        for index, row in val.iterrows():
             # access data using column names
            audio_fname = row['audio_filename']
            shutil.copy(os.path.join(mavd_path, "audio_segments", "validate", audio_fname), os.path.join(out_val_dir,audio_fname))

        print("Validation files copied")
        '''
        print("Copying test files...")
        for index, row in test.iterrows():
             # access data using column names
            audio_fname = row['audio_filename']
            shutil.copy(os.path.join(mavd_path, "audio_segments","test", audio_fname), os.path.join(out_test_dir,audio_fname))

        print("Test files copied")
        '''
        print("Copying train files (this may take a while)...")

        for index, row in train.iterrows():
             # access data using column names
            audio_fname = row['audio_filename']
            shutil.copy(os.path.join(mavd_path, "audio_segments","train", audio_fname), os.path.join(out_train_dir,audio_fname))

        print("Train files copied!")


import numpy as np
import matplotlib.pyplot as plt

N = 5
menMeans = (20, 35, 30, 35, 27)
womenMeans = (25, 32, 34, 20, 25)
ind = np.arange(N)
width = 0.35

p1 = plt.bar(ind, menMeans, width, color = "r")
p2 = plt.bar(ind, womenMeans, width, color = "y", bottom = menMeans)

plt.ylabel("Scores")
plt.title("Scores by group and sex")
plt.xticks(ind + width / 2, ("G1", "G2", "G3", "G4", "G5"))
plt.yticks(np.arange(0, 81, 10))
plt.legend((p1[0], p2[0]), ("Men", "Women"))

plt.show()


# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
from typing import Union

import blist
import numpy as np
import torch

from torchsde._brownian import utils
from torchsde._brownian.base_brownian import Brownian

class BrownianPath(Brownian):
    """Fast Brownian motion with all queries stored in a list and uses local search.

    Trades in memory for speed.

    To use:
    >>> bm = BrownianPath(t0=0.0, w0=torch.zeros(4, 1))
    >>> bm(0.5)
    tensor([[ 0.0733],
            [-0.5692],
            [ 0.1872],
            [-0.3889]])
    """

    def __init__(self, t0: Union[float, torch.Tensor], w0: torch.Tensor, window_size: int = 8):
        """Initialize Brownian path.

        Args:
            t0: Initial time.
            w0: Initial state.
            window_size: Size of the window around the last query for local search.
        """
        super(BrownianPath, self).__init__()
        if not utils.is_scalar(t0):
            raise ValueError('Initial time t0 should be a float or 0-d torch.Tensor.')

        t0 = float(t0)
        self._ts = blist.blist()
        self._ws = blist.blist()
        self._ts.append(t0)
        self._ws.append(w0)

        self._last_idx = 0
        self._window_size = window_size

    def __call__(self, t):
        t = float(t)
        if t == self._ts[-1]:
            idx = len(self._ts) - 1
            w = self._ws[idx]
            found = True
        elif t == self._ts[0]:
            idx = 0
            w = self._ws[idx]
            found = True
        elif t > self._ts[-1]:
            idx = len(self._ts)
            t0 = self._ts[-1]
            t1 = t
            dw = torch.randn_like(self._ws[0]) * math.sqrt(t1 - t0)
            w = self._ws[-1] + dw
            found = False
        elif t < self._ts[0]:
            idx = 0
            t0 = t
            t1 = self._ts[0]
            dw = torch.randn_like(self._ws[0]) * math.sqrt(t1 - t0)
            w = self._ws[0] - dw
            found = False
        else:
            # Try local neighborhood of last query.
            left_idx = max(0, self._last_idx - self._window_size)
            right_idx = min(len(self._ts), self._last_idx + self._window_size)
            idx, w, found = utils.search(self._ts[left_idx:right_idx], self._ws[left_idx:right_idx], t)
            if w is None:  # t not within range of local neighborhood.
                # t must be within range of self._ts. This is ensured by the entering logic.
                idx, w, found = utils.search(self._ts, self._ws, t)
            else:  # Convert idx to be in full range.
                idx = idx + left_idx

        if not found:
            self._ts.insert(idx, t)
            self._ws.insert(idx, w)
        self._last_idx = idx
        return w

    def insert(self, t, w):
        """Insert scalar time and tensor Brownian motion state into list.

        The method silently replaces the original value if t is already in the list, and returns the old value.
        Otherwise returns None.

        The method should only be used for testing purposes.
        """
        t = float(t)
        if t > self._ts[-1]:
            idx = len(self._ts)
            old = None
        elif t < self._ts[0]:
            idx = 0
            old = None
        else:
            # TODO: Replace with `torch.searchsorted` when torch==1.7.0 releases.
            #  Also need to make sure we use tensor dt.
            idx = np.searchsorted(self._ts, t)
            if t == self._ts[idx]:
                old = self._ws[idx]
            else:
                old = None

        self._ts.insert(idx, t)
        self._ws.insert(idx, w)
        self._last_idx = idx
        return old

    def __repr__(self):
        return f"BrownianPath(t0={self._ts[0]:.3f}, t1={self._ts[-1]:.3f})"

    def to(self, *args, **kwargs):
        self._ws = utils.blist_to(self._ws, *args, **kwargs)

    @property
    def dtype(self):
        return self._ws[0].dtype

    @property
    def device(self):
        return self._ws[0].device

    @property
    def shape(self):
        return self._ws[0].size()

    def size(self):
        return self._ws[0].size()

    def __len__(self):
        return len(self._ts)

    def get_cache(self):
        return {'ts': self._ts, 'ws': self._ws}


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
'''
@author: zzy
@file: get_uid.py
@time: 2019/10/5 23:03
'''
import json
import os

from pymongo import MongoClient
class Get_uid:
    def __init__(self):
        # 数据库配置信息
        self.conf = json.load(open(os.path.abspath("mongo_server.json")))

    def conn_mongo(self,col_name):
        '''
        函数功能：连接MongoDB数据库，获取uid
        :param col_name: 数据库col名
        :return: uids可迭代对象
        '''
        while True:
            try:
                # 连接服务器
                conf = self.conf
                client = MongoClient(conf["server_ip"],int(conf["server_port"]))
                # 连接zzy数据库，账号密码认证
                db = client[conf["mongo_name"]]
                db.authenticate(conf["mongo_user"],conf["mongo_passwd"])
                mycol = db[col_name]
                # 查询u集合里的所有uid
                uids = mycol.find({},{"_id":0,"uid":1})
                client.close()
                break
            except Exception as e:
                print("查询uid异常",e)
                continue
        return uids

    def get_movie_uid(self):
        '''
        函数功能：获取影视明星uid
        :return: uid列表
        '''
        uids = self.conn_mongo("movie_uid")
        uid_dict = [uid for uid in uids] # [{'uid': '1223178222'}, {'uid': '1918601565'}, {'uid': '1259110474'}]
        uid_list = [k["uid"] for k in uid_dict] # ['1223178222', '1918601565', '1259110474', '1304194202']
        return uid_list

    def get_music_uid(self):
        '''
        函数功能：获取唱歌明星uid
        :return: uid列表
        '''
        uids = self.conn_mongo("music_uid")
        uid_dict = [uid for uid in uids] # [{'uid': '1223178222'}, {'uid': '1918601565'}, {'uid': '1259110474'}]
        uid_list = [k["uid"] for k in uid_dict] # ['1223178222', '1918601565', '1259110474', '1304194202']
        return uid_list

    def get_game_uid(self):
        '''
        函数功能：获取游戏主播uid
        :return: uid列表
        '''
        uids = self.conn_mongo("music_uid")
        uid_dict = [uid for uid in uids] # [{'uid': '1223178222'}, {'uid': '1918601565'}, {'uid': '1259110474'}]
        uid_list = [k["uid"] for k in uid_dict] # ['1223178222', '1918601565', '1259110474', '1304194202']
        return uid_list

    def get_sports_uid(self):
        '''
        函数功能：获取体育明星uid
        :return: uid列表
        '''
        uids = self.conn_mongo("sports_uid")
        uid_dict = [uid for uid in uids] # [{'uid': '1223178222'}, {'uid': '1918601565'}, {'uid': '1259110474'}]
        uid_list = [k["uid"] for k in uid_dict] # ['1223178222', '1918601565', '1259110474', '1304194202']
        return uid_list

if __name__ == '__main__':
    uid =Get_uid()
    print(uid.get_movie_uid())


#!/usr/bin/python3
# Fab file to launch the whole show
from datetime import datetime
from fabric.api import local
import os.path
from fabric.api import put
from fabric.api import run
from fabric.api import env

env.hosts = ["35.237.36.104", "34.74.62.199"]

def do_pack():
    """Create and return a tar.gz obj of web_static"""
    t = datetime.utcnow()
    # concat file name
    ret = "versions/web_static_{}{}{}{}{}{}.tgz".format(t.year,
                                                        t.month,
                                                        t.day,
                                                        t.hour,
                                                        t.minute,
                                                        t.second)
    # if the pathname isn't versions...
    if os.path.isdir("versions") is False:
        # and the mkdir fails...
        if local("mkdir -p versions").failed is True:
            return None
    # simulat. run and conditional
    if local("tar -cvzf {} web_static".format(ret)).failed is True:
        return None
    # come back and build print statement
    return ret

def do_deploy(archive_path):
    """This does deploy"""

    # check that passed file exists
    if os.path.isfile(archive_path) is False:
        return False

    # get file name from file path
    file_name = archive_path.split("/")[-1]
    # remove file extension
    no_ext = file_name.split(".")[0]

    # put the file on the server
    if put(archive_path, "/tmp/{}".format(file_name)).failed is True:
        return False
    # mkdir for new release
    if run("mkdir -p /data/web_static/releases/{}/".
           format(no_ext)).failed is True:
        return False
    # decompress file in tmp but move it to releases folder
    if run("tar -xzf /tmp/{} -C /data/web_static/releases/{}/".
           format(file_name, no_ext)).failed is True:
        return False
    # remove archive file
    if run("rm /tmp/{}".format(file_name)).failed is True:
        return False
    # Move the contents of the folder up one folder
    if run("mv /data/web_static/releases/{}/web_static/*"
           " /data/web_static/releases/{}/"
           .format(no_ext, no_ext)).failed is True:
        return False
    # kill empty folder
    if run("rm -fr /data/web_static/releases/{}/web_static".
           format(no_ext)).failed is True:
        return False
    # delete symbolic link
    if run("rm -fr /data/web_static/current").failed is True:
        return False
    # create new symbolic link
    if run("ln -s /data/web_static/releases/{}/ /data/web_static/current".
           format(no_ext)).failed is True:
        return False
    return True

def deploy():
    """This deploys the whole deal"""
    arch = do_pack()
    if arch is None:
        return False
    return do_deploy(arch)


"""Loader that loads YouTube transcript."""
from __future__ import annotations

from pathlib import Path
from typing import Any, Dict, List, Optional

from pydantic import root_validator
from pydantic.dataclasses import dataclass

from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader

SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]

@dataclass
class GoogleApiClient:
    """A Generic Google Api Client.

    To use, you should have the ``google_auth_oauthlib,youtube_transcript_api,google``
    python package installed.
    As the google api expects credentials you need to set up a google account and
    register your Service. "https://developers.google.com/docs/api/quickstart/python"

    Example:
        .. code-block:: python

            from langchain.document_loaders import GoogleApiClient
            google_api_client = GoogleApiClient(
                service_account_path=Path("path_to_your_sec_file.json")
            )

    """

    credentials_path: Path = Path.home() / ".credentials" / "credentials.json"
    service_account_path: Path = Path.home() / ".credentials" / "credentials.json"
    token_path: Path = Path.home() / ".credentials" / "token.json"

    def __post_init__(self) -> None:
        self.creds = self._load_credentials()

    @root_validator
    def validate_channel_or_videoIds_is_set(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""

        if not values.get("credentials_path") and not values.get(
            "service_account_path"
        ):
            raise ValueError("Must specify either channel_name or video_ids")
        return values

    def _load_credentials(self) -> Any:
        """Load credentials."""
        # Adapted from https://developers.google.com/drive/api/v3/quickstart/python
        try:
            from google.auth.transport.requests import Request
            from google.oauth2 import service_account
            from google.oauth2.credentials import Credentials
            from google_auth_oauthlib.flow import InstalledAppFlow
            from youtube_transcript_api import YouTubeTranscriptApi  # noqa: F401
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib"
                "youtube-transcript-api`"
                "to use the Google Drive loader"
            )

        creds = None
        if self.service_account_path.exists():
            return service_account.Credentials.from_service_account_file(
                str(self.service_account_path)
            )
        if self.token_path.exists():
            creds = Credentials.from_authorized_user_file(str(self.token_path), SCOPES)

        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    str(self.credentials_path), SCOPES
                )
                creds = flow.run_local_server(port=0)
            with open(self.token_path, "w") as token:
                token.write(creds.to_json())

        return creds

class YoutubeLoader(BaseLoader):
    """Loader that loads Youtube transcripts."""

    def __init__(
        self, video_id: str, add_video_info: bool = False, language: str = "en"
    ):
        """Initialize with YouTube video ID."""
        self.video_id = video_id
        self.add_video_info = add_video_info
        self.language = language

    @classmethod
    def from_youtube_url(cls, youtube_url: str, **kwargs: Any) -> YoutubeLoader:
        """Given youtube URL, load video."""
        video_id = youtube_url.split("youtube.com/watch?v=")[-1]
        return cls(video_id, **kwargs)

    def load(self) -> List[Document]:
        """Load documents."""
        try:
            from youtube_transcript_api import NoTranscriptFound, YouTubeTranscriptApi
        except ImportError:
            raise ImportError(
                "Could not import youtube_transcript_api python package. "
                "Please install it with `pip install youtube-transcript-api`."
            )

        metadata = {"source": self.video_id}

        if self.add_video_info:
            # Get more video meta info
            # Such as title, description, thumbnail url, publish_date
            video_info = self._get_video_info()
            metadata.update(video_info)

        transcript_list = YouTubeTranscriptApi.list_transcripts(self.video_id)
        try:
            transcript = transcript_list.find_transcript([self.language])
        except NoTranscriptFound:
            en_transcript = transcript_list.find_transcript(["en"])
            transcript = en_transcript.translate(self.language)

        transcript_pieces = transcript.fetch()

        transcript = " ".join([t["text"].strip(" ") for t in transcript_pieces])

        return [Document(page_content=transcript, metadata=metadata)]

    def _get_video_info(self) -> dict:
        """Get important video information.

        Components are:
            - title
            - description
            - thumbnail url,
            - publish_date
            - channel_author
            - and more.
        """
        try:
            from pytube import YouTube

        except ImportError:
            raise ImportError(
                "Could not import pytube python package. "
                "Please install it with `pip install pytube`."
            )
        yt = YouTube(f"https://www.youtube.com/watch?v={self.video_id}")
        video_info = {
            "title": yt.title,
            "description": yt.description,
            "view_count": yt.views,
            "thumbnail_url": yt.thumbnail_url,
            "publish_date": yt.publish_date,
            "length": yt.length,
            "author": yt.author,
        }
        return video_info

@dataclass
class GoogleApiYoutubeLoader(BaseLoader):
    """Loader that loads all Videos from a Channel

    To use, you should have the ``googleapiclient,youtube_transcript_api``
    python package installed.
    As the service needs a google_api_client, you first have to initialize
    the GoogleApiClient.

    Additionally you have to either provide a channel name or a list of videoids
    "https://developers.google.com/docs/api/quickstart/python"

    Example:
        .. code-block:: python

            from langchain.document_loaders import GoogleApiClient
            from langchain.document_loaders import GoogleApiYoutubeLoader
            google_api_client = GoogleApiClient(
                service_account_path=Path("path_to_your_sec_file.json")
            )
            loader = GoogleApiYoutubeLoader(
                google_api_client=google_api_client,
                channel_name = "CodeAesthetic"
            )
            load.load()

    """

    google_api_client: GoogleApiClient
    channel_name: Optional[str] = None
    video_ids: Optional[List[str]] = None
    add_video_info: bool = True
    captions_language: str = "en"

    def __post_init__(self) -> None:
        self.youtube_client = self._build_youtube_client(self.google_api_client.creds)

    def _build_youtube_client(self, creds: Any) -> Any:
        try:
            from googleapiclient.discovery import build
            from youtube_transcript_api import YouTubeTranscriptApi  # noqa: F401
        except ImportError:
            raise ImportError(
                "You must run"
                "`pip install --upgrade "
                "google-api-python-client google-auth-httplib2 "
                "google-auth-oauthlib"
                "youtube-transcript-api`"
                "to use the Google Drive loader"
            )

        return build("youtube", "v3", credentials=creds)

    @root_validator
    def validate_channel_or_videoIds_is_set(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate that either folder_id or document_ids is set, but not both."""
        if not values.get("channel_name") and not values.get("video_ids"):
            raise ValueError("Must specify either channel_name or video_ids")
        return values

    def _get_transcripe_for_video_id(self, video_id: str) -> str:
        from youtube_transcript_api import NoTranscriptFound, YouTubeTranscriptApi

        transcript_list = YouTubeTranscriptApi.list_transcripts(self.video_ids)
        try:
            transcript = transcript_list.find_transcript([self.captions_language])
        except NoTranscriptFound:
            en_transcript = transcript_list.find_transcript(["en"])
            transcript = en_transcript.translate(self.captions_language)

        transcript_pieces = transcript.fetch()
        return " ".join([t["text"].strip(" ") for t in transcript_pieces])

    def _get_document_for_video_id(self, video_id: str, **kwargs: Any) -> Document:
        captions = self._get_transcripe_for_video_id(video_id)
        video_response = (
            self.youtube_client.videos()
            .list(
                part="id,snippet",
                id=video_id,
            )
            .execute()
        )
        return Document(
            page_content=captions,
            metadata=video_response.get("items")[0],
        )

    def _get_channel_id(self, channel_name: str) -> str:
        request = self.youtube_client.search().list(
            part="id",
            q=channel_name,
            type="channel",
            maxResults=1,  # we only need one result since channel names are unique
        )
        response = request.execute()
        channel_id = response["items"][0]["id"]["channelId"]
        return channel_id

    def _get_document_for_channel(self, channel: str, **kwargs: Any) -> List[Document]:
        channel_id = self._get_channel_id(channel)
        request = self.youtube_client.search().list(
            part="id,snippet",
            channelId=channel_id,
            maxResults=50,  # adjust this value to retrieve more or fewer videos
        )
        video_ids = []
        while request is not None:
            response = request.execute()

            # Add each video ID to the list
            for item in response["items"]:
                if not item["id"].get("videoId"):
                    continue
                meta_data = {"videoId": item["id"]["videoId"]}
                if self.add_video_info:
                    item["snippet"].pop("thumbnails")
                    meta_data.update(item["snippet"])
                video_ids.append(
                    Document(
                        page_content=self._get_transcripe_for_video_id(
                            item["id"]["videoId"]
                        ),
                        metadata=meta_data,
                    )
                )
            request = self.youtube_client.search().list_next(request, response)

        return video_ids

    def load(self) -> List[Document]:
        """Load documents."""
        document_list = []
        if self.channel_name:
            document_list.extend(self._get_document_for_channel(self.channel_name))
        elif self.video_ids:
            document_list.extend(
                [
                    self._get_document_for_video_id(video_id)
                    for video_id in self.video_ids
                ]
            )
        else:
            raise ValueError("Must specify either channel_name or video_ids")
        return document_list


#!/usr/bin/python
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bootstrapper for running a VM script.

Args:
test-files-gcs-dir: The Cloud Storage location containing the test files.
  This dir of test files must contain a test.py containing the test logic.
"""
import logging
import os
import subprocess
import urllib2

def GetMetadataAttribute(attribute):
  url = 'http://metadata/computeMetadata/v1/instance/attributes/%s' % attribute
  request = urllib2.Request(url)
  request.add_unredirected_header('Metadata-Flavor', 'Google')
  return urllib2.urlopen(request).read()

def Bootstrap():
  """Get test files, run test."""
  try:
    logging.info('Starting bootstrap.py.')
    test_gcs_dir = GetMetadataAttribute('test_files_gcs_dir')
    test_script = GetMetadataAttribute('test_script')
    test_dir = '/test_files'
    full_test_script = os.path.join(test_dir, test_script)
    subprocess.check_call(['mkdir', test_dir])
    subprocess.check_call(
        ['gsutil', '-m', 'cp', '-r', os.path.join(test_gcs_dir, '*'),
         test_dir])
    logging.info('Making test script %s executable.', full_test_script)
    subprocess.check_call(['chmod', '+x', test_script], cwd=test_dir)
    logging.info('Running %s.', full_test_script)
    subprocess.check_call([full_test_script], cwd=test_dir)
  except Exception as e:
    print('TestFailed: error: ' + str(e))

if __name__ == '__main__':
  logging.basicConfig(level=logging.DEBUG)
  Bootstrap()


# This should fix some "reset" bugs and keep memory cleaner

# by Sov

import App
import MissionLib
from Custom.DS9FX.DS9FXLib import DS9FXSets, GalaxyCharts

bDeleteSets = 1

def KillSets(pObject, pEvent):
    pDisallow = GalaxyCharts.IsIncompatibleOn()
    if pDisallow:
        return 0
    pPlayer = MissionLib.GetPlayer()
    if not pPlayer:
        return 0
    pSet = pPlayer.GetContainingSet()
    if not pSet:
        return 0
    pShip = App.ShipClass_Cast(pEvent.GetDestination())
    if not pShip:
        return 0
    if not pShip.GetObjID() == pPlayer.GetObjID():
        return 0

    # A bad fix implementation, but will do
    pSequence = App.TGSequence_Create()
    pAction = App.TGScriptAction_Create(__name__, "KillSetsDelay")
    pSequence.AddAction(pAction, None, 3)
    pSequence.Play()

def KillSetsDelay(pAction):
    global bDeleteSets

    if not bDeleteSets:
        return 0

    pPlayer = MissionLib.GetPlayer()
    if not pPlayer:
        return 0

    pSet = pPlayer.GetContainingSet()
    if not pSet:
        return 0

    lSets = DS9FXSets.lDS9FXSets

    if not pSet.GetName() in lSets:
        return 0

    for s in lSets:
        if s == pSet.GetName():
            continue
        try:
            App.g_kSetManager.DeleteSet(s)
        except:
            pass

    return 0

def DisallowSetKilling():
    global bDeleteSets
    bDeleteSets = 0

    # Using timers when a new QB Mission is loading causes event "mixings", so better use sequence which doesn't cause any errors
    pSequence = App.TGSequence_Create()
    pAction = App.TGScriptAction_Create(__name__, "AllowSetKilling")
    pSequence.AddAction(pAction, None, 10)
    pSequence.Play()

def AllowSetKilling(pAction):
    global bDeleteSets
    bDeleteSets = 1

    return 0


# Author: Jake Vanderplas -- <vanderplas@astro.washington.edu>

print(__doc__)

from time import time

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import NullFormatter
import numpy as np
from sklearn import manifold, datasets

# Next line to silence pyflakes. This import is needed.
Axes3D

n_points = 1000
data = np.genfromtxt('swissRollpoints.txt')
X = data[:,0:3]
color = data[:,3]
n_neighbors = 10
n_components = 2

fig = plt.figure(figsize=(15, 8))
plt.suptitle("Manifold Learning with %i points, %i neighbors"
             % (1000, n_neighbors), fontsize=14)

ax = fig.add_subplot(251, projection='3d')
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=color, cmap=plt.cm.Spectral)
ax.view_init(4, -72)

methods = ['ltsa']
labels = ['LLE', 'LTSA', 'Hessian LLE', 'Modified LLE']

for i, method in enumerate(methods):
    t0 = time()
    Y = manifold.LocallyLinearEmbedding(n_neighbors, n_components,
                                        eigen_solver='auto',
                                        method=method).fit_transform(X)
    t1 = time()
    print("%s: %.2g sec" % (methods[i], t1 - t0))

    ax = fig.add_subplot(252 + i)
    plt.scatter(Y[:, 0], Y[:, 1], c=color, cmap=plt.cm.Spectral)
    plt.title("%s (%.2g sec)" % (labels[i], t1 - t0))
    ax.xaxis.set_major_formatter(NullFormatter())
    ax.yaxis.set_major_formatter(NullFormatter())
    plt.axis('tight')

plt.show()


from misc import *

class Healthbar(pygame.sprite.Sprite):
    def __init__(self, player):
        super().__init__()

        self.image = pygame.Surface((BLOCK_WIDTH*4 + 10, 40))
        self.image.fill(colors["GRAY"])
        temp = pygame.Surface((BLOCK_WIDTH * 4, 30))
        temp.fill(colors["RED"])
        self.image.blit(temp, (5,5))
        self.background = pygame.Surface(self.image.get_size())
        self.background.blit(self.image, (0,0))
        self.rect = self.image.get_rect()
        self.max_health = player.MAX_HEALTH

    def update(self, health):
        self.image.blit(self.background, (0,0))
        if health > 0:
            temp = pygame.Surface((int(health/self.max_health*BLOCK_WIDTH*4), 30))
            temp.fill(colors["GREEN"])
            self.image.blit(temp, (5,5))

class HUD():
    '''HUD class'''
    def __init__(self, character):
        self.healthbar = Healthbar(character)

    def update(self, character):
        self.healthbar.update(int(character.health))


# coding: utf-8
# python 3.5
# Nov 27, 2018

class Solution:
    def selfDividingNumbers(self, left, right):
        """
        left, right: int
        return: list of int
        """

        output = []
        for num in range(left, right+1):
            str_ = str(num)
            if '0' in str_:
                continue

            flag = True
            for s in str_:
                i = int(s)
                if num % i != 0:
                    flag = False
                    break

            if flag:
                output.append(num)

        return output

solu = Solution()
print(solu.selfDividingNumbers(1, 22))