import os
import pytest
import subprocess
import time

from common import create_volume_and_write_data
from common import volume_name  # NOQA
from common import get_self_host_id
from common import wait_for_volume_detached
from common import wait_for_volume_healthy
from common import wait_for_volume_degraded
from common import wait_for_volume_replica_count
from common import write_volume_random_data
from common import check_volume_data
from common import get_longhorn_api_client
from common import prepare_pod_with_data_in_mb
from common import update_statefulset_manifests
from common import create_storage_class
from common import create_and_wait_statefulset
from common import get_statefulset_pod_info
from common import get_apps_api_client
from common import delete_and_wait_pod
from common import wait_statefulset
from common import create_pvc_spec
from common import create_and_wait_pod
from common import get_pod_data_md5sum
from common import pod_make, csi_pv, pvc # NOQA
from common import statefulset # NOQA
from common import storage_class # NOQA
from common import SETTING_AUTO_SALVAGE
from common import generate_random_data
from common import VOLUME_RWTEST_SIZE
from common import write_pod_volume_data
from common import read_volume_data
from common import settings_reset # NOQA
from common import client, core_api # NOQA
from common import SETTING_DISABLE_REVISION_COUNTER
from common import update_setting
from common import delete_replica_on_test_node
from common import get_volume_engine

@pytest.fixture
def upgrade_longhorn_repo_url(request):
    return request.config.getoption("--upgrade-lh-repo-url")

@pytest.fixture
def upgrade_longhorn_repo_branch(request):
    return request.config.getoption("--upgrade-lh-repo-branch")

@pytest.fixture
def upgrade_longhorn_manager_image(request):
    return request.config.getoption("--upgrade-lh-manager-image")

@pytest.fixture
def upgrade_longhorn_engine_image(request):
    return request.config.getoption("--upgrade-lh-engine-image")

@pytest.fixture
def upgrade_longhorn_instance_manager_image(request):
    return request.config.getoption("--upgrade-lh-instance-manager-image")

@pytest.fixture
def upgrade_longhorn_share_manager_image(request):
    return request.config.getoption("--upgrade-lh-share-manager-image")

@pytest.fixture
def upgrade_longhorn_backing_image_manager_image(request):
    return request.config.getoption("--upgrade-lh-backing-image-manager-image")

def get_longhorn_upgrade_type():
    return [os.environ.get('LONGHORN_UPGRADE_TYPE', '')]

@pytest.fixture(params=get_longhorn_upgrade_type())
def longhorn_upgrade_type():
    # add parameter "from_stable" or "from_transient" to test_upgrade test case
    # to distinguish them in the junit report.
    pass

def longhorn_upgrade(longhorn_repo_url,
                     longhorn_repo_branch,
                     longhorn_manager_image,
                     longhorn_engine_image,
                     longhorn_instance_manager_image,
                     longhorn_share_manager_image,
                     longhorn_backing_image_manager_image):

    command = "../scripts/upgrade-longhorn.sh"
    process = subprocess.Popen([command,
                                longhorn_repo_url,
                                longhorn_repo_branch,
                                longhorn_manager_image,
                                longhorn_engine_image,
                                longhorn_instance_manager_image,
                                longhorn_share_manager_image,
                                longhorn_backing_image_manager_image],
                               shell=False)
    process.wait()
    if process.returncode == 0:
        longhorn_upgraded = True

    else:
        longhorn_upgraded = False

    return longhorn_upgraded

@pytest.mark.upgrade  # NOQA
def test_upgrade(longhorn_upgrade_type,
                 upgrade_longhorn_repo_url,
                 upgrade_longhorn_repo_branch,
                 upgrade_longhorn_manager_image,
                 upgrade_longhorn_engine_image,
                 upgrade_longhorn_instance_manager_image,
                 upgrade_longhorn_share_manager_image,
                 upgrade_longhorn_backing_image_manager_image,
                 client, core_api, volume_name, csi_pv, # NOQA
                 pvc, pod_make, statefulset, storage_class): # NOQA
    """
    Test Longhorn upgrade

    TODO
    The test will cover both volume has revision counter enabled and
    disabled cases.

    Prerequisite:
      - Disable Auto Salvage Setting

    1. Find the upgrade image tag
    2. Create a volume, generate and write data into the volume.
        1. Create vol_revision_enabled with revision counter enabled case.
        2. Create vol_revision_disabled with revision counter disabled case.
        3. Create vol_rebuild for replica rebuilding after system upgrade
           & engine live upgrade
    3. Create a Pod using a volume, generate and write data
    4. Create a StatefulSet with 2 replicas
       generate and write data to their volumes
    5. Keep all volumes attached
    6. Upgrade Longhorn system.
    7. Check Pod and StatefulSet didn't restart after system upgrade
    8. Check all volumes data
    9. Write all volumes data after system upgrade
    10. Check data written to all volumes after system upgrade
    11. Detach the vol_revision_enabled & vol_revision_disabled,
        and Delete Pod, and StatefulSet to detach theirvolumes
    12. Upgrade all volumes engine images
    13. Attach the volume, and recreate Pod, and StatefulSet
    14. Verify the volume's engine image has been upgraded
    15. Check All volumes data
    16. Delete one replica for vol_rebuild to trigger the rebuilding
    17. Verify the vol_rebuild is still healthy
    """
    longhorn_repo_url = upgrade_longhorn_repo_url
    longhorn_repo_branch = upgrade_longhorn_repo_branch
    longhorn_manager_image = upgrade_longhorn_manager_image
    longhorn_engine_image = upgrade_longhorn_engine_image
    longhorn_instance_manager_image = upgrade_longhorn_instance_manager_image
    longhorn_share_manager_image = upgrade_longhorn_share_manager_image
    longhorn_backing_image_manager_image = \
        upgrade_longhorn_backing_image_manager_image

    host_id = get_self_host_id()
    pod_data_path = "/data/test"

    # Disable Auto Salvage Setting
    update_setting(client, SETTING_AUTO_SALVAGE, "false")

    # 2-1 Create vol_revision_enabled with revision counter enabled
    # attached to a node
    update_setting(client, SETTING_DISABLE_REVISION_COUNTER, "false")
    vol_revision_enabled_name = 'vol-revision-enabled'
    vol_revision_enabled, vol_revision_enabled_data_before_sys_upgrade = \
        create_volume_and_write_data(client, vol_revision_enabled_name)

    # 2-2 Create vol_revision_disabled with revision counter disable
    # attached to a node
    update_setting(client, SETTING_DISABLE_REVISION_COUNTER, "true")
    vol_revision_disabled_name = 'vol-revision-disabled'
    vol_revision_disabled, vol_revision_disabled_data_before_sys_upgrade = \
        create_volume_and_write_data(client, vol_revision_disabled_name)

    # 2-3 Create vol_rebuild for replica rebuilding after system upgrade
    # & engine live upgrade
    vol_rebuild_name = 'vol-rebuild'
    vol_rebuild, vol_rebuild_data_before_sys_upgrade = \
        create_volume_and_write_data(client, vol_rebuild_name)

    # Create Volume used by Pod
    pod_volume_name = 'lh-vol-pod-test'
    pod_name, pv_name, pvc_name, pod_md5sum = \
        prepare_pod_with_data_in_mb(client, core_api, csi_pv, pvc,
                                    pod_make, pod_volume_name,
                                    data_path=pod_data_path,
                                    add_liveness_probe=False)

    # Create multiple volumes used by StatefulSet
    statefulset_name = 'statefulset-upgrade-test'
    update_statefulset_manifests(statefulset,
                                 storage_class,
                                 statefulset_name)
    create_storage_class(storage_class)
    create_and_wait_statefulset(statefulset)
    statefulset_pod_info = get_statefulset_pod_info(core_api, statefulset)

    for sspod_info in statefulset_pod_info:
        sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
        write_pod_volume_data(core_api,
                              sspod_info['pod_name'],
                              sspod_info['data'])
    # upgrade Longhorn manager
    assert longhorn_upgrade(longhorn_repo_url,
                            longhorn_repo_branch,
                            longhorn_manager_image,
                            longhorn_engine_image,
                            longhorn_instance_manager_image,
                            longhorn_share_manager_image,
                            longhorn_backing_image_manager_image)

    client = get_longhorn_api_client()

    # wait for 1 minute before checking pod restarts
    time.sleep(60)

    # Check Pod and StatefulSet didn't restart after upgrade
    pod = core_api.read_namespaced_pod(name=pod_name,
                                       namespace='default')
    assert pod.status.container_statuses[0].restart_count == 0

    for sspod_info in statefulset_pod_info:
        sspod = core_api.read_namespaced_pod(name=sspod_info['pod_name'],
                                             namespace='default')
        assert \
            sspod.status.container_statuses[0].restart_count == 0

    # Check all volumes data after system upgrade
    check_volume_data(vol_revision_enabled,
                      vol_revision_enabled_data_before_sys_upgrade)
    check_volume_data(vol_revision_disabled,
                      vol_revision_disabled_data_before_sys_upgrade)
    check_volume_data(vol_rebuild,
                      vol_rebuild_data_before_sys_upgrade)

    for sspod_info in statefulset_pod_info:
        resp = read_volume_data(core_api, sspod_info['pod_name'])
        assert resp == sspod_info['data']

    res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
    assert res_pod_md5sum == pod_md5sum

    # Write data to all volumes after system upgrade
    for sspod_info in statefulset_pod_info:
        sspod_info['data'] = generate_random_data(VOLUME_RWTEST_SIZE)
        write_pod_volume_data(core_api,
                              sspod_info['pod_name'],
                              sspod_info['data'])

    vol_revision_enabled_data_after_sys_upgrade = \
        write_volume_random_data(vol_revision_enabled)
    vol_revision_disabled_data_after_sys_upgrade = \
        write_volume_random_data(vol_revision_disabled)
    vol_rebuild_data_after_sys_upgrade = \
        write_volume_random_data(vol_rebuild)

    # Check data written to all volumes
    for sspod_info in statefulset_pod_info:
        resp = read_volume_data(core_api, sspod_info['pod_name'])
        assert resp == sspod_info['data']

    check_volume_data(vol_revision_enabled,
                      vol_revision_enabled_data_after_sys_upgrade)
    check_volume_data(vol_revision_disabled,
                      vol_revision_disabled_data_after_sys_upgrade)
    check_volume_data(vol_rebuild,
                      vol_rebuild_data_after_sys_upgrade)

    # Detach the vol_revision_enabled & vol_revision_disabled,
    # and Delete Pod, and StatefulSet to detach theirvolumes

    statefulset['spec']['replicas'] = replicas = 0
    apps_api = get_apps_api_client()

    apps_api.patch_namespaced_stateful_set(
        name=statefulset_name,
        namespace='default',
        body={
            'spec': {
                'replicas': replicas
            }
        })

    delete_and_wait_pod(core_api, pod_name)

    # Upgrade all volumes engine images
    volumes = client.list_volume()
    for v in volumes:
        if v.name != vol_rebuild_name:
            volume = client.by_id_volume(v.name)
            volume.detach(hostId="")
            wait_for_volume_detached(client, v.name)

    engineimages = client.list_engine_image()
    for ei in engineimages:
        if ei.image == longhorn_engine_image:
            new_ei = ei

    for v in volumes:
        volume = client.by_id_volume(v.name)
        volume.engineUpgrade(image=new_ei.image)

    # Recreate Pod, and StatefulSet
    statefulset['spec']['replicas'] = replicas = 2
    apps_api = get_apps_api_client()

    apps_api.patch_namespaced_stateful_set(
        name=statefulset_name,
        namespace='default',
        body={
            'spec': {
                'replicas': replicas
            }
        })

    wait_statefulset(statefulset)

    pod = pod_make(name=pod_name)
    pod['spec']['volumes'] = [create_pvc_spec(pvc_name)]
    create_and_wait_pod(core_api, pod)

    # Attach the volume
    for v in volumes:
        if v.name == vol_revision_enabled_name or \
                v.name == vol_revision_disabled_name:
            volume = client.by_id_volume(v.name)
            volume.attach(hostId=host_id)
            wait_for_volume_healthy(client, v.name)

    # Verify volume's engine image has been upgraded
    for v in volumes:
        volume = client.by_id_volume(v.name)
        engine = get_volume_engine(volume)
        assert engine.engineImage == new_ei.image
        assert engine.currentImage == new_ei.image

    # Check All volumes data
    for sspod_info in statefulset_pod_info:
        resp = read_volume_data(core_api, sspod_info['pod_name'])
        assert resp == sspod_info['data']

    res_pod_md5sum = get_pod_data_md5sum(core_api, pod_name, pod_data_path)
    assert res_pod_md5sum == pod_md5sum

    check_volume_data(vol_revision_enabled,
                      vol_revision_enabled_data_after_sys_upgrade)
    check_volume_data(vol_revision_disabled,
                      vol_revision_disabled_data_after_sys_upgrade)
    check_volume_data(vol_rebuild,
                      vol_rebuild_data_after_sys_upgrade)

    # Delete one healthy replica for vol_rebuild to trigger the rebuilding
    delete_replica_on_test_node(client, vol_rebuild_name)
    # Make sure vol_rebuild replica is deleted
    replica_count = 2
    vol_rebuild = wait_for_volume_replica_count(client, vol_rebuild_name,
                                                replica_count)
    # vol_rebuild will become degraded and start replica rebuilding
    # Wait for replica rebuilding to complete
    # Verify the vol_rebuild is still healthy
    vol_rebuild = wait_for_volume_degraded(client, vol_rebuild_name)
    assert vol_rebuild.robustness == "degraded"
    vol_rebuild = wait_for_volume_healthy(client, vol_rebuild_name)
    assert vol_rebuild.robustness == "healthy"
    assert len(vol_rebuild.replicas) == 3

# Need add this test case into test_upgrade()
# https://github.com/longhorn/longhorn/issues/4726
@pytest.mark.skip(reason="TODO")  # NOQA
def test_upgrade_with_auto_upgrade_latest_engine_enabled():
    """
    1. Deploy Longhorn stable version
    2. Set Concurrent Automatic Engine Upgrade Per Node Limit to > 0
    3. Create a volume and attach it
    4. Deploy longhornio/longhorn-engine:master-head and wait for it
       to be deployed. This step is to make sure to expose the race condition
       that Longhorn tries auto engine upgrade while the new default IM is
       still starting
    5. Upgrade Longhorn to master-head
    6. Observe volume engine image upgrade success.
    """


from pprint import pprint
import json

import pytest

from models.contact_model import Contact
from models.experience_model import Experience, Month
from models.resume_model import Resume
from models.resume_section_model import ResumeSection
from models.tag_model import Tag
from models.tag_item_model import TagItem
from models.program_contact_model import ProgramContact
from models.session_model import UserSession

from .test_api import post_request, POSTS

# This only works because the script is at the top level
from resources.trello_utils import (
    query_board_data,
    Board,
    BoardList,
    Card,
)

TALENT_INTAKE_BOARDS = {
    'local': '5e37744114d9d01a03ddbcfe',
    'dev': '5e25dae2e640be7e5248a3e6',
    'production': '5ddd741f5cc43e2b21346dbb'
}

TALENT_INTAKE_LABELS = [
    'New', 'Fellowship', 'Mayoral Fellowship', 'PFP', 'PA', 'JHU - Carey'
]

TALENT_INTAKE_ATTACHMENTS = ['Profile', 'Full Response']

def check_labels(board_id, expected_labels):
    board = Board(query_board_data(board_id))
    labels = board.labels['name'].keys()
    print(labels)
    for label in expected_labels:
        assert label in labels

def test_talent_intake_board_labels(app):
    local_board_id = TALENT_INTAKE_BOARDS['local']
    check_labels(local_board_id, TALENT_INTAKE_LABELS)
    prod_board_id = TALENT_INTAKE_BOARDS['production']
    check_labels(prod_board_id, TALENT_INTAKE_LABELS)
    dev_board_id = TALENT_INTAKE_BOARDS['dev']
    check_labels(dev_board_id, TALENT_INTAKE_LABELS)

def test_talent_intake(app):
    mimetype = 'application/x-www-form-urlencoded'
    headers = {
        'Content-Type': mimetype,
        'Accept': mimetype,
    }

    card_id = '5e4af2d6fc3c0954ff187ddc'

    contact = Contact.query.get(123)
    board_id = TALENT_INTAKE_BOARDS['local']
    board = Board(query_board_data(board_id))
    card = board.cards.get(card_id)
    started_list = board.lists['stage'][1]

    for attachment in TALENT_INTAKE_ATTACHMENTS:
        response = card.remove_attachment(attachment)
    card.set_labels(label_names=['New', 'Fellowship'])
    card.move_card(started_list)

    board = Board(query_board_data(board_id))
    card = board.cards.get(card_id)

    assert card.attachments == {}
    assert card.label_names == ['New', 'Fellowship']
    assert card.stage == 1
    assert contact.stage == 1

    url = '/api/form-assembly/talent-app/'
    data = (
        'contact_id=123&program_id=1&first_name=Billy&last_name=Daly&email=billy%40example.com&phone=9085784622&street1=2401+Liberty+Heights+Ave&street2=Suite+2730&city=Baltimore&state=MD&postal_code=21215&equity=Race&effectiveness=Effectiveness&programs%5B0%5D=Fellowship&programs%5B1%5D=JHU+-+Carey&programs%5B2%5D=Mayoral+Fellowship&programs%5B3%5D=PFP&programs%5B4%5D=PA&programs%5B5%5D=I%27d+like+some+help+figuring+this+out&mayoral_eligible=Yes&experience=0-2+years&capabilities%5B0%5D=Advocacy+and+Public+Policy&capabilities%5B1%5D=Community+Engagement+and+Outreach&capabilities%5B2%5D=Data+Analysis&capabilities%5B3%5D=Fundraising+and+Development&capabilities%5B4%5D=Marketing+and+Public+Relations&alum_radio=No&race_all=American+Indian+or+Alaskan+Native&gender=Male&pronouns=He%2FHim&response_id=160140910'
    )
    with app.test_client() as client:
        response = client.post(url, data=data, headers=headers)
        pprint(response.json)
        assert response.status_code == 201
        board = Board(query_board_data(board_id))
        card = board.cards.get(card_id)
        contact = Contact.query.get(123)
        assert card.stage == 2
        assert contact.stage == 2
        assert 'New' in card.label_names
        for label in TALENT_INTAKE_LABELS[1:]:
            assert label in card.label_names
        assert 'Race' in card.desc
        assert '- Data Analysis\n' in card.desc
        assert '0-2 years' in card.desc
        assert '**Mayoral Fellowship:** Yes' in card.desc
        assert '**JHU - Carey:** N/A' in card.desc
        pprint(card.attachments)
        assert card.attachments['Profile']['url'] == (
            'https://app.baltimorecorps.org/profile/123'
        )
        assert card.attachments['Full Response']['url'] == (
            'https://app.formassembly.com/responses/view/160140910'
        )
        data = json.loads(response.data)['data']

def test_resubmit_approved_app(app):
    mimetype = 'application/x-www-form-urlencoded'
    headers = {
        'Content-Type': mimetype,
        'Accept': mimetype,
    }

    card_id = '5e4af2d6fc3c0954ff187ddc'

    board_id = TALENT_INTAKE_BOARDS['local']
    board = Board(query_board_data(board_id))
    card = board.cards.get(card_id)
    approved_list = board.lists['stage'][3]
    card.move_card(approved_list)

    board = Board(query_board_data(board_id))
    card = board.cards.get(card_id)
    assert card.stage == 3

    url = '/api/form-assembly/talent-app/'
    data = (
        'contact_id=123&program_id=1&first_name=Billy&last_name=Daly&email=billy%40example.com&phone=9085784622&street1=2401+Liberty+Heights+Ave&street2=Suite+2730&city=Baltimore&state=MD&postal_code=21215&equity=Race&effectiveness=Effectiveness&programs%5B0%5D=Fellowship&programs%5B1%5D=JHU+-+Carey&programs%5B2%5D=Mayoral+Fellowship&programs%5B3%5D=PFP&programs%5B4%5D=PA&programs%5B5%5D=I%27d+like+some+help+figuring+this+out&mayoral_eligible=Yes&experience=0-2+years&capabilities%5B0%5D=Advocacy+and+Public+Policy&capabilities%5B1%5D=Community+Engagement+and+Outreach&capabilities%5B2%5D=Data+Analysis&capabilities%5B3%5D=Fundraising+and+Development&capabilities%5B4%5D=Marketing+and+Public+Relations&alum_radio=No&race_all=American+Indian+or+Alaskan+Native&gender=Male&pronouns=He%2FHim&response_id=160140910'
    )

    with app.test_client() as client:
        response = client.post(url, data=data, headers=headers)
        pprint(response.json)
        assert response.status_code == 400
        message = json.loads(response.data)['message']
        assert message == 'Application has already been submitted'

# TODO: Add trello specific checks
def test_create_program_contact_with_contact(app):
    mimetype = 'application/json'
    headers = {
        'Content-Type': mimetype,
        'Accept': mimetype,
        'Authorization': 'Bearer test-valid|0123456789',
    }
    with app.test_client() as client:
        response = client.post('/api/contacts/',
                               data=json.dumps(POSTS['contact']),
                               headers=headers)

        assert response.status_code == 201
        data = json.loads(response.data)['data']
        assert len(data) > 0
        assert data['id'] is not None
        id_ = data['id']

        program_contacts = Contact.query.get(id_).programs
        assert len(program_contacts) == 1
        assert program_contacts[0].program_id == 1
        assert program_contacts[0].stage == 1
        assert program_contacts[0].program.name == 'Place for Purpose'
        assert program_contacts[0].is_active == True
        assert program_contacts[0].is_approved == False

# TODO: Add trello specific checks
def test_post_contact(app):
    mimetype = 'application/json'
    headers = {
        'Content-Type': mimetype,
        'Accept': mimetype,
        'Authorization': 'Bearer test-valid|0123456789',
    }
    with app.test_client() as client:
        response = client.post('/api/contacts/',
                               data=json.dumps(POSTS['contact']),
                               headers=headers)
        assert response.status_code == 201
        set_cookie = response.headers.get('set-cookie')
        assert set_cookie is not None
        assert set_cookie.find('HttpOnly;') is not -1
        # Note: Can't test "secure" due to non-https connection
        contact = Contact.query.filter_by(account_id='test-valid|0123456789').first()
        assert contact.first_name == 'Tester'
        assert contact.email == 'testerb@example.com'
        assert contact.profile.years_exp is None

        assert UserSession.query.filter_by(contact_id=contact.id).first()

# TODO: Add trello specific checks
def test_post_duplicate_contact(app):
    mimetype = 'application/json'
    headers = {
        'Content-Type': mimetype,
        'Accept': mimetype,
        'Authorization': 'Bearer test-valid|0123456789abcdefabcdefff',
    }

    contact_data = POSTS['contact'].copy()
    contact_data['account_id'] = 'test-valid|0123456789abcdefabcdefff'

    with app.test_client() as client:
        response = client.post('/api/contacts/',
                               data=json.dumps(contact_data),
                               headers=headers)
        assert response.status_code == 400
        message = json.loads(response.data)['message']
        assert message == 'A contact with this account already exists'


"""
A working demo using KMeans

"""

import numpy as np
import scipy.io as sio
import matplotlib.pyplot as plt

data = sio.loadmat('data.mat')
X = np.array(data['X'])

from KMeans import KMeans

k = 3
est = KMeans(k)

c = est.train(X)

colors=np.array(['green', 'red', 'blue'])
# lets plot on matplotlib
for i in range(k):
    x = X[np.where(c == i)[0]]
    plt.scatter(x[:, 0], x[:, 1], color=colors[i])

# plt.savefig('clustering_example.png')
plt.show()


# encoding: utf8
from fp_growth import find_frequent_itemsets

ff=open("fp_list_1.txt", 'w')
if __name__ == '__main__':
    from optparse import OptionParser
    import csv

    p = OptionParser(usage='%prog data_file')
    p.add_option('-s', '--minimum-support', dest='minsup', type='int',
        help='Minimum itemset support (default: 2)')  ## 設定指令的option -s int(設定minimum support)
    p.set_defaults(minsup=2)

    options, args = p.parse_args()
    if len(args) < 1:
        p.error('must provide the path to a CSV file to read')

    f = open(args[0])
    try:
        for itemset, support in find_frequent_itemsets(csv.reader(f), options.minsup, True):
            if len(itemset) >= 5:
                printtext = ','.join(itemset) + " : " + str(support)
                ff.write(printtext+"\n")
    finally:
        f.close()
ff.close()


#!/usr/bin/python
from shark import *

def process_rel(rel, rel_name_prefix, length):
    write_rel("/mnt/shared/ssn_hybrid/data", rel_name_prefix + "_indexes.csv", rel, "idx")
    original_ordering = get_original_ordering(rel)
    encoded = encode_as_flags(project(original_ordering, [1]), length)
    write_rel("/mnt/shared/ssn_hybrid/data", rel_name_prefix + "_encoded.csv", encoded, "idx")
    write_rel("/mnt/shared/ssn_hybrid/data", rel_name_prefix + "_original_ordering.csv",
              project(original_ordering, [0]),
              "idx")

if __name__ == "__main__":
    print("start python")
    in1_keys = read_rel('/mnt/shared/ssn_hybrid/data/in1_keys.csv')
    in1_keys_indexed = project_indeces(in1_keys)
    in2_keys = read_rel('/mnt/shared/ssn_hybrid/data/in2_keys.csv')
    in2_keys_indexed = project_indeces(in2_keys)
    joined_indexes = join(in1_keys_indexed, in2_keys_indexed, 1, 1)
    in1_indexes = project(joined_indexes, [1])
    in2_indexes = project(joined_indexes, [2])
    process_rel(in1_indexes, "in1", len(in1_keys))
    process_rel(in2_indexes, "in2", len(in2_keys))
    assert len(in1_indexes) == len(in2_indexes)
    write_rel("/mnt/shared/ssn_hybrid/data", "num_lookups.csv", [[len(in1_indexes)]], "idx")
    print("done python")


#!/usr/bin/python -W ignore
########################################################## {COPYRIGHT-TOP} ###
# Licensed Materials - Property of IBM
# 5737-I32
#
# (C) Copyright IBM Corp. 2019
#
# US Government Users Restricted Rights - Use, duplication, or
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp.
########################################################## {COPYRIGHT-END} ###

import logging
import os
import sys
import re

ENCODING = 'utf-8'

class DocumentRetrievalFactory:
    """Factory class to create the right sort of retrieval object."""

    @staticmethod
    def create(application, key):
        """Lookup connection and create required type."""
        platform, client = application.connections.get((key.datasource, key.cluster), (None, None))

        if platform and client:
            if platform == 'COS':
                return DocumentRetrievalCOS(client)
            elif platform == 'NFS':
                return DocumentRetrievalNFS(client)
            elif platform == 'Spectrum Scale':
                return DocumentRetrievalScale(client)
            elif platform == 'Spectrum Scale Local':
                return DocumentRetrievalLocalScale(client)
        return None

class DocumentRetrievalBase():
    """A class to retrieve a document via a Spectrum Discover Connection."""

    def __init__(self, client):
        """Constructor."""
        # Instantiate logger
        loglevels = {'INFO': logging.INFO, 'DEBUG': logging.DEBUG,
                     'ERROR': logging.ERROR, 'WARNING': logging.WARNING}
        log_level = os.environ.get('LOG_LEVEL', 'DEBUG')
        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        logging.basicConfig(stream=sys.stdout,
                            format=log_format,
                            level=loglevels[log_level])
        self.logger = logging.getLogger(__name__)

        self.client = client

    def get_document(self, key):
        """
        To be implemented by the child class.

        Receives a document key that uniquely identifies a particular
        object, and is responsible for retrieving that object and
        returning the filepath as a string.

        Returns None if object was not able to be retrieved.
        """
        self.logger.warning("get_document has not been implemented for this class")
        return None

    def get_headers(self, key):
        """
        To be implemented by the child class.

        Receives a document bucket and key that uniquely identifies a particular
        cos object, and is responsible for retrieving that objects headers.

        Returns None if object was not able to be retrieved.
        """
        self.logger.warning("get_headers has not been implemented for this class")
        return None

    def cleanup_document(self):
        """
        To be implemented by the child class.

        Perform cleanup and/or remove any tmp files as needed.

        Returns None.
        """
        self.logger.warning("cleanup_document has not been implemented for this class")

    def create_file_path(self, prefix, path):
        """Return the tmpfile_path with filetype as string."""
        self.logger.debug('filepath prefix: %s, path: %s', prefix, path.decode(ENCODING))
        if '.' in path.decode(ENCODING):
            return prefix + '.' + path.decode(ENCODING).split('.')[-1]
        return prefix

class DocumentRetrievalCOS(DocumentRetrievalBase):
    """Create a COS document class.

    This will act appropriately upon COS documents that have been downloaded as temp files.
    """

    filepath = None

    def get_document(self, key):
        """Return document filepath."""
        content = None
        self.filepath = None

        if self.client:
            obj = self.client.get_object(Bucket=key.datasource, Key=key.path.decode(ENCODING).split('/', 1)[1])
            content = obj['Body'].read()
            self.filepath = self.create_file_path('/tmp/cosfile_' + str(os.getpid()), key.path)
            with open(self.filepath, 'w', encoding=ENCODING) as file:
                file.write(content.decode(ENCODING, 'replace'))
        else:
            self.logger.error('Could not access file %s', key.path.decode(ENCODING))

        return self.filepath

    def get_headers(self, key):
        """Return the COS HTTPHeaders for a key, bucket."""
        http_headers = None

        if self.client:
            vault, doc_name = key.path.decode(ENCODING).split('/', 1)
            headers = self.client.head_object(Bucket=vault, Key=doc_name)
            try:
                http_headers = headers['ResponseMetadata']['HTTPHeaders']
            except KeyError:
                self.logger.error('Could not access header metadata information for file %s: %s', key.path.decode(ENCODING), headers)
        else:
            self.logger.error('Could not access client')
        return http_headers

    def cleanup_document(self):
        """Cleanup files as needed."""
        self.logger.debug("COS: Attempting to delete file: %s", self.filepath)

        try:
            os.remove(self.filepath)
        except FileNotFoundError:
            self.logger.debug('No file: %s to delete.', self.filepath)

class DocumentRetrievalNFS(DocumentRetrievalBase):
    """Create a NFS document class.

    This will act appropriately upon nfs documents.
    No need to download or cleanup since accessed direclty upon the mount point.
    """

    filepath = None

    def get_document(self, key):
        """Return document filepath."""
        self.filepath = None

        if self.client:
            mount_path_prefix = self.client['additional_info']['local_mount']
            source_path_prefix = self.client['mount_point']
            self.filepath = re.sub('^' + source_path_prefix, mount_path_prefix, key.path.decode(ENCODING))
        else:
            print('No document match')

        return self.filepath

    def cleanup_document(self):
        """Cleanup files as needed."""
        self.logger.debug("NFS: Not doing any cleanup.")

class DocumentRetrievalScale(DocumentRetrievalBase):
    """Create a Spectrum Scale document class.

    This will act appropriately upon Scale documents that have been downloaded as temp files.
    """

    filepath = None

    def get_document(self, key):
        """Return document filepath."""
        self.filepath = None

        if self.client:
            try:
                self.filepath = self.create_file_path('/tmp/scalefile_' + str(os.getpid()), key.path)
                self.logger.debug(self.filepath)
                self.client.get(key.path, self.filepath)
            except UnicodeDecodeError:
                self.logger.error('Could not decode file %s', key.path.decode(ENCODING))
            except FileNotFoundError:
                self.logger.error('Could not find file %s', key.path.decode(ENCODING))
            except OSError: # Seen when scale nsd disk is down and file is not accessible
                self.logger.error('Could not transfer file %s', key.path.decode(ENCODING))

        else:
            print('No document match')

        return self.filepath

    def cleanup_document(self):
        """Cleanup files as needed."""
        self.logger.debug("SCALE: Attempting to delete file: %s", self.filepath)

        try:
            os.remove(self.filepath)
        except FileNotFoundError:
            self.logger.debug('No file: %s to delete.', self.filepath)

class DocumentRetrievalLocalScale(DocumentRetrievalBase):
    """Create a Local Spectrum Scale document class.

    This will act appropriately upon nfs documents.
    No need to download or cleanup since accessed direclty upon the mount point.
    """

    def get_document(self, key):
        """Return Document Key."""
        return key.path.decode(ENCODING)

    def cleanup_document(self):
        """Cleanup files as needed."""
        self.logger.debug("Scale Local: Not doing any cleanup.")

class DocumentKey(object):
    """A class to identify a unique document on a Spectrum Discover Connection."""

    def __init__(self, doc):
        """Init."""
        self.fkey = doc['fkey']
        self.datasource = doc['datasource']
        self.cluster = doc['cluster']
        self.path = doc['path'].encode(ENCODING)
        # a unique identifier for the connection this document belongs to.
        self.id = self.datasource + ':' + self.cluster

    def __str__(self):
        """Return string formatted doc info."""
        return "{}/{} -> {} (fkey: {})".format(self.datasource, self.cluster, self.path, self.fkey)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.contrib import messages
from django.shortcuts import render, redirect
from .models import User, Message, Comment
import re
EMAILREG = re.compile(r'^[a-zA-Z0-9\.\+_-]+@[a-zA-Z0-9\._-]+\.[a-zA-Z]*$')

# Create your views here.
def index(request):
    return render(request, "wall/index.html")

def wall(request):
    context = {
        "user_messages": Message.objects.all().order_by('id').reverse(),
        "user_comments": Comment.objects.all().order_by('id'),
    }
    return render(request, "wall/wall.html", context)

def post_message(request):
    if request.method == 'POST':
        if request.POST["wall_posts"] == "message_submit":
            response = Message.objects.new_message_input(request.POST, request.session['user']['id'])
        if request.POST["wall_posts"] == "comment_submit":
            response = Comment.objects.new_comment_input(request.POST, request.session['user']['id'])
    return redirect('wall:wall')

def user(request, id):
    user_check = User.objects.filter(id=id)
    if user_check:
        context = {
            "user": User.objects.get(id=id),
        }
        return render(request, "wall/user.html", context)
    else:
        return redirect('wall:all_users')

def all_users(request):
    context = {
        "all": User.objects.all(),
    }
    return render(request, "wall/allusers.html", context)

def entrance(request):
    if request.method == "POST":
        if request.POST["gate"] == "register":
            response = User.objects.check_create(request.POST)
        elif request.POST["gate"] == "login":
            response = User.objects.check_login(request.POST)
        if not response[0]:
            for error in response[1]:
                messages.error(request, error[1])
            return redirect('wall:index')
        else:
            request.session['user'] = {
            "id": response[1].id,
            "first_name": response[1].first_name,
            "last_name": response[1].last_name,
            }
            return redirect('wall:wall')
    return redirect('wall:index')

def user_update(request):
    user_id = str(request.session['user']['id'])
    if request.method == "POST":
        if request.POST["update_option"] == "profile":
            response = User.objects.check_update(request.POST, request.session['user']['id'])
        if request.POST["update_option"] == "password":
            response = User.objects.check_password(request.POST, request.session['user']['id'])
        if not response[0]:
            for error in response[1]:
                messages.error(request, error[1])
            return redirect('/'+user_id)
    return redirect('/'+user_id)

def user_delete(request):
    user_id = request.session['user']['id']
    User.objects.filter(id=user_id).delete()
    return redirect('wall:logout')

def logout(request):
    request.session.clear()
    return redirect('wall:index')


import simpy
import datetime
import time
import random
import os
import pandas
import openpyxl as opxl
import win32com.client as win32
import numpy as np
from openpyxl.utils.dataframe import dataframe_to_rows

class Simulation:

    def __init__(self,serveTime,nServers,nEmployees,lunchTime):
        self.startTime = time.time()
        self.serveTime = int(serveTime)
        self.nServers = int(nServers)
        self.nEmployees = int(nEmployees)
        self.lunchTime = lunchTime
        self.data = {"Arrival Time":[],"Queue Time":[],"Inter-Arrival":[]}

    def customer(self,env, server, TBA, serveTime):

        # Simulate a period of time before the arrival of the next customer
        yield env.timeout(TBA)

        # Record time of arrival
        arriveTime = env.now
        self.data["Arrival Time"].append(arriveTime)

        # Join the queue
        with server.request() as req:
            # Wait if resource is being used
            yield req

            # Get served
            queueTime = (env.now - arriveTime)
            self.data["Queue Time"].append(queueTime)

            yield env.timeout(serveTime)

    def distribute(self,mInterval,length):
        intervals = list(random.expovariate(1.0/mInterval) for i in range(length))
        self.data["Inter-Arrival"] = intervals
        return list(np.cumsum(intervals))

    def runSimulation(self):
        # Initialise environment and resource
        env = simpy.Environment()
        cafe = simpy.Resource(env, capacity=self.nServers)

        # Generate customers
        meanTBA = int(self.lunchTime)/(int(self.nEmployees)-1)
        arrivalTimes = self.distribute(meanTBA,self.nEmployees)
        for i in range(self.nEmployees):
            env.process(self.customer(env, cafe, arrivalTimes[i], self.serveTime))

        # Run simulation
        env.run()

    def exportData(self):
        simDay = datetime.datetime.now()
        sheetName = simDay.strftime("%H %M %S %f")
        direcName = os.path.dirname(os.path.abspath(__file__)) + "\\" + fileName

        # Export data collected to Excel
        try:
            wb = opxl.load_workbook(filename=fileName)
        except FileNotFoundError:
            new_wb = opxl.Workbook()
            new_wb.save("%s.xlsx" % self.dist)
            wb = new_wb

        ws = wb.create_sheet(title="%s" % sheetName)
        df = pandas.DataFrame.from_dict(self.data)

        for row in dataframe_to_rows(df, index=True, header=True):
            ws.append(row)

        # Format table
        ws['A1'] = 'ID #'

        #Summarise data
        ws['H2'] = 'Summary'
        ws['H3'],ws['I3'] = 'Avg. Queue Time',np.mean(self.data["Queue Time"])
        ws['H4'],ws['I4'] = '# of Servers',self.nServers
        ws['H5'],ws['I5'] = 'Serve Time (s)',self.serveTime
        ws['H6'],ws['I6'] = 'Lunch Time (hrs)',round(self.lunchTime/3600,1)
        ws['H7'],ws['I7'] = 'Mean TBA (s)',np.mean(self.data["Inter-Arrival"])
        ws['H8'],ws['I8'] = 'Spread',np.std(self.data["Queue Time"])

        # Save workbook
        wb.save(fileName)

        # Autofit columns
        excel = win32.gencache.EnsureDispatch('Excel.Application')
        wb = excel.Workbooks.Open(direcName)
        ws = wb.Worksheets(sheetName)
        ws.Columns.AutoFit()
        wb.Save()
        excel.Application.Quit()

    def printRuntime(self):
        print('RUNTIME: --- %.2f seconds ---' % (time.time()-self.startTime))

    def run(self):
        self.runSimulation()
        self.exportData()
        self.printRuntime()

if __name__ == '__main__':

    startTime = time.time()

	#Input Data
    serveTimes = [5,10,15,30,45]
    numServers = [1,2,3,5]
    nEmployees = 1250
    lunchTimes = [1800,3600,7200]

    fileName = "FCFS.xlsx"
    for sTime in serveTimes:
        for num in numServers:
            for lunchTime in lunchTimes:
                s = Simulation(sTime,num,nEmployees,lunchTime)
                s.run()

    print("TOTAL RUNTIME: %.2f seconds" % (time.time() - startTime))


from django.contrib.postgres.search import (SearchQuery, SearchRank,
                                            SearchVector)
from django.shortcuts import render

from analyzer.utils.stemmer import stemm
from cards.models import Meal

from .forms import SearchForm

def search(request, ingredients_correction=None, meal_url_correction=None):
    meals = Meal.objects.all().order_by('-created')[0:3]
    form = SearchForm()
    query = None
    results = []
    if request.method == 'POST':
        if 'query' in request.POST:
            form = SearchForm(request.POST)
            if form.is_valid():
                query = form.cleaned_data['query']
                token = stemm(query)
                results = Meal.objects.annotate(search=SearchVector('meal_name', 'ingredients', 'tokens'),).filter(search=token[0])
                for word in token:
                    results = results | Meal.objects.annotate(search=SearchVector('meal_name', 'ingredients', 'tokens'),).filter(search=word)
    else:
        form = SearchForm()
    return render(request, 'search/search.html', {'form': form, 'query': query, 'results': results, 'meals': meals})


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Feb  7 21:32:00 2021

@author: vriesdejelmer
"""

import csv
import torch

import numpy as np
import matplotlib.collections as mcoll
import matplotlib.path as mpath
import matplotlib.pyplot as plt

def writeProperties(new_vars, old_vars, data_dir):

    with open(data_dir + 'properties.csv', 'w') as csv_file:
        writer = csv.writer(csv_file)
        for key in set(new_vars) - set(old_vars):
            if key != 'old_vars':
                writer.writerow([key, new_vars[key]])

def getDevice():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)
    if not (str(device) == "cuda"):
        print("WE'RE NOT RUNNING ON CUDA")

    return device

def colorline(x, y, z=None, cmap=plt.get_cmap('copper'), norm=plt.Normalize(0.0, 1.0), linewidth=5, alpha=1.0):
    """
    http://nbviewer.ipython.org/github/dpsanders/matplotlib-examples/blob/master/colorline.ipynb
    http://matplotlib.org/examples/pylab_examples/multicolored_line.html
    Plot a colored line with coordinates x and y
    Optionally specify colors in the array z
    Optionally specify a colormap, a norm function and a line width
    """

    # Default colors equally spaced on [0,1]:
    if z is None:
        z = np.linspace(0.0, 1.0, len(x))

    # Special case if a single number:
    if not hasattr(z, "__iter__"):  # to check for numerical input -- this is a hack
        z = np.array([z])

    z = np.asarray(z)

    segments = make_segments(x, y)
    lc = mcoll.LineCollection(segments, array=z, cmap=cmap, norm=norm,
                              linewidth=linewidth, alpha=alpha)

    ax = plt.gca()
    ax.add_collection(lc)

    return lc

def make_segments(x, y):
    """
    Create list of line segments from x and y coordinates, in the correct format
    for LineCollection: an array of the form numlines x (points per line) x 2 (x
    and y) array
    """
    points = np.array([x, y]).T.reshape(-1, 1, 2)
    segments = np.concatenate([points[:-1], points[1:]], axis=1)
    return segments

    #convert cartesian coordinates to polar
def cart2pol(x, y):
    rho = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return(rho, phi)

    #convert proportions to cartesian coords on the unit circle
def prop2unit(prop):
    x_coord = np.cos(prop*2*np.pi)
    y_coord = np.sin(prop*2*np.pi)
    return(x_coord, y_coord)


import sys
import os
import subprocess
from subprocess import Popen, PIPE, STDOUT
import shlex
import glob

 #~ /WORKS/cmarchet/scripts/headersPB.pl cluster10168.fasta > cluster10168_PB.fasta &
#~ /WORKS/cmarchet/bin/DAZZ_DB/fasta2DB -v db_cluster cluster10168_PB.fasta
 #~ /daligner db_cluster.db db_cluster.db
#~ /WORKS/cmarchet/bin/DALIGNER-1.0/LAsort db_cluster.db_cluster.*
#~ /WORKS/cmarchet/bin/DALIGNER-1.0/LAmerge db_cluster.db_cluster_final.las  db_cluster.db_cluster*.S.las
#~ /WORKS/cmarchet/bin/daccord-0.0.10-release-20170526170720-x86_64-etch-linux-gnu/bin/daccord db_cluster.db_cluster_final.las db_cluster.db > result_cluster.fasta 2> null

#garder la plus longue séquence du fasta final

# launch subprocess
def subprocessLauncher(cmd, argstdout=None, argstderr=None,	 argstdin=None):
	args = shlex.split(cmd)
	p = subprocess.call(args, stdin = argstdin, stdout = argstdout, stderr = argstderr)
	return p

def dazzdb_daligner(clusterFa, clusterName, clusterDir,PIPELINE_DIR):
	#clusterName = clusterDir + "/" + clusterFa
	file_name = os.path.join(clusterDir,clusterFa)
	#cmdHeaders = "LongReadPipeline/scripts/headersPB.pl " + clusterDir + "/" + clusterFa
	cmdHeaders = "{}/scripts/headersPB.pl {}".format(PIPELINE_DIR, file_name)
	#fastaPacbioHeader = clusterDir + "/" + clusterName + "_PB.fasta"
	fastaPacbioHeader = os.path.join(clusterDir,"{0}_PB.fasta".format(clusterName))
	fastaPacbioHeader_to_write = open(fastaPacbioHeader, 'w') #dazzdb looks for files ending
	# exactly by .fasta (no .fa)
	p = subprocessLauncher(cmdHeaders, fastaPacbioHeader_to_write)
	fastaPacbioHeader_to_write.close()
	# launch dazzdb
	#clusterDB = clusterDir + "/db_" + clusterName
	clusterDB = os.path.abspath(os.path.join(clusterDir,"db_{0}".format(clusterName)))
	nameDB = os.path.splitext(os.path.basename(clusterDB))[0]
	assert nameDB.startswith("db_")
	# cmdDazzDB = "./DAZZ_DB/fasta2DB -v " + clusterDB + " " + fastaPacbioHeader
	cmdDazzDB = "{}/pbdagcon/DAZZ_DB/fasta2DB -v {} {}".format(PIPELINE_DIR, nameDB,fastaPacbioHeader)
	p = subprocessLauncher(cmdDazzDB)
	# daligner has to be launched from its directory
	# daligner binary copied from pddagcon submodules
	#cmdDaligner = "./DALIGNER/daligner " + "db_" + clusterName + " " + "db_" + clusterName
	cmdDaligner = "{0}/pbdagcon/DALIGNER/daligner {1} {1}".format(PIPELINE_DIR, clusterDB)
	p = subprocessLauncher(cmdDaligner)
	#DAZZ_DB_output = clusterDir + "/db_" + clusterName + ".db_" + clusterName + "*"
	daligner_output = "{}.db_{}*".format(clusterDB, clusterName)
	#cmdsort = "./DALIGNER-1.0/LAsort " + clusterDir + "/db_" + clusterName + ".db_" + clusterName + "*"
	cmdsort = "{}/pbdagcon/DALIGNER/LAsort {}".format(PIPELINE_DIR, daligner_output)
	subprocess.check_output(['bash','-c', cmdsort])
	#final_las = clusterDir + "/db_" + clusterName + "_final.las"
	#S_las = clusterDir + "/db_" + clusterName + ".db_" + clusterName + "*.S.las"
	final_las = "{0}_final.las".format(clusterDB)
	S_las = "{0}.S.las".format(daligner_output)
	#cmdmerge = "./DALIGNER-1.0/LAmerge "  + clusterDir + "/db_" + clusterName + "_final.las " + clusterDir + "/db_" +\
	# 		  clusterName + ".db_" + clusterName + "*.S.las"
	cmdmerge = "{}/pbdagcon/DALIGNER/LAmerge {} {}".format(PIPELINE_DIR, final_las, S_las)
	subprocess.check_output(['bash','-c', cmdmerge])

#~ def pbdagcon(suffix):
	#~ outDagcon = open("corrected_by_PBDagCon" + suffix + ".fa", 'w')
	#~ cmdDagcon = "dazcon -ox -j 4 -s db_simulatedLR" + suffix + ".db -a db_simulatedLR" + suffix + ".las"
	#~ p = subprocessLauncher(cmdDagcon, outDagcon)
	#~ outDagcon.close()

def daccord(clusterName, clusterDir, PIPELINE_DIR):
	outDaccord = "corrected_by_daccord_{0}.fa".format(clusterName)
	#outDaccord = open("corrected_by_daccord_" + clusterName + ".fa", 'w')
	clusterDB = os.path.abspath(os.path.join(clusterDir, "db_{0}.db".format(clusterName)))
	final_las = "{0}_final.las".format(clusterDB.replace(".db",""))
	outDaccord_to_write = open(outDaccord, "w")
	#cmdDaccord = "./daccord/bin/daccord " + clusterDir + "/db_" +\
	#			 clusterName + "_final.las " + clusterDir + "/db_" + clusterName + ".db"
	cmdDaccord = "{}/daccord/bin/daccord {} {}".format(PIPELINE_DIR, final_las, clusterDB)
	p = subprocessLauncher(cmdDaccord, outDaccord_to_write)
	outDaccord_to_write.close()

def main():
	clusterDir = os.path.dirname(os.path.abspath(sys.argv[1]))
	with open(sys.argv[1]) as listFasta:
		for fasta in listFasta:
			print("This is fasta file : {0}".format(fasta))
			nameFasta = fasta.split(".")[0] #id of the cluster
			dazzdb_daligner(fasta.rstrip(), nameFasta, clusterDir)
			daccord(nameFasta, clusterDir)
			for f in glob.glob('*.las'):
				os.remove(f)
			for f in glob.glob("db_*"):
				os.remove(f)
			os.remove(clusterDir + "/" + nameFasta + "_PB.fasta")

if __name__ == "__main__":
	main()


graph = {
    "x1": [("x2",10), ('x3',3)],
    'x2': [('x1',10)],
    'x3': [('x1',3),('x2',4),('x4',13)],
    'x4': [('x3',13)]
}

def find_shortest_path(graph, start, end, path=[]):
    print("starting", start, end)
    path = path + [start]
    if start == end:
        print("   returned 0")
        return path, 0
    if start not in graph:
        print("   returned None")
        return None, None
    shortest_path = None
    shortest_dist = None

    for node, dist in graph[start]:
        if node not in path:
            newpath, dist1 = find_shortest_path(graph, node, end, path)
            if newpath is not None:
                print("   going from",start,"to", node, dist, "+", dist1, "=", dist+dist1)
                dist1 += dist
                if not shortest_dist or dist1 < shortest_dist:
                    shortest_path = newpath
                    shortest_dist = dist1

    print("returning", shortest_path, shortest_dist, "if started from", start)
    return shortest_path, shortest_dist

print("Result", find_shortest_path(graph, 'x1', 'x2'))


def summation(ll, ul):
	sum = 0
	for i in range(ll, ul+1):
		if ( i % 2 != 0):
			sum = sum + i
	return sum

def main():
	ll = input("Enter the lower limit :")
	ul = input("Enter the upper limit :")
	print ("The sum of all odd numbers in the given range is %d"%(summation(ll, ul)))

if __name__ == "__main__":
	main()


''' This program is responsible for
instantiating the GUI, connecting
to the microphone and speaker, and
managing the program flow '''

import intake, understander, processor, generator
import os

# tts
from gtts import gTTS
from io import BytesIO
import pygame
from pygame import mixer
from tempfile import TemporaryFile

# stt
import speech_recognition as sr

UNK = -1
COUNSELOR = 0
CLIENT = 1

class App():
	def __init__(self):
		# First, instantiate all the relevant objects
		self.client = Client()
		self.counselor = Counselor()
		self.conversation = Conversation(self.counselor, self.client)

class Conversation():
	def __init__(self, counselor, client):
		self.statements = []	# a list of statements
		self.counselor = counselor
		self.client = client

	def start(self):
		self.counselor.introduce(self.client)
		self.counselor.speak("Ok, " + self.client.name + ". Let's start by learning about who you are.")
		self.client.personality = intake.conduct_interview()
		self.counselor.speak("Great, thank you. This will be helpful as I listen to you.")
		self.counselor.speak("Why don't you tell me what's on your mind?")

		while True:
			# Get the user's response and parse it
			response = Statement(input("[YOU]\t\t"), self.client)

			# append it to the conversation
			self.statements.append(response)

			# select a microskill
			self.most_recent_statement().microskill_response = processor.pick_microskill(self)

			# generate it
			processor.test_all_microskills(self)
			#processor.generate_response(self.most_recent_statement().microskill_response, self)

	def most_recent_statement(self):
		return self.statements[len(self.statements) - 1]

class Statement():
	def __init__(self, raw_msg, speaker):
		self.speaker = speaker
		self.raw_msg = raw_msg	# each sentence in the statement
		self.parsed_msg = understander.parse_msg(self)	# parsed_msg is a nlp() object
		self.noun_phrases = [chunk.text for chunk in self.parsed_msg.noun_chunks]
		self.msg_emotions = understander.get_sentence_emotions(self.raw_msg)
		self.msg_topic = understander.get_topic(self.raw_msg, self.parsed_msg)
		self.msg_category = understander.get_category(self.raw_msg, self.parsed_msg)
		self.microskill_response = None

class Client():
	def __init__(self):
		self.name = ""
		self.personality = []	# a dict of trait-->value
		self.mood = {}			# a dict of descriptor-->value

	def is_client(self):
		return True

class Counselor():
	def __init__(self):
		self.persona = []		# the same as client personality
								# except more variable and
								# complementary to the client

	def is_client(self):
		return False

	def speak(self, sentence):
		print("[COUNSELOR]\t" + sentence)

		#tts = gTTS(sentence, lang="en")
		#mixer.init()

		#sf = TemporaryFile()
		#tts.write_to_fp(sf)
		#sf.seek(0)
		#mixer.music.load(sf)
		#mixer.music.play()

		#while mixer.music.get_busy():
		#	pygame.time.wait(100)

	def listen(self):
		print("[YOU]\t\t", end="")

		transcription = input("")# self.get_speech()

		#print(transcription)

		return transcription

	def introduce(self, client):
		self.speak("Hello. Welcome to the Virtual Counselor.")
		self.speak("What's your name?")
		client.name = self.listen()

	def get_speech(self):
		r = sr.Recognizer()
		mic = sr.Microphone()

		with mic as source:
			r.adjust_for_ambient_noise(source)
			audio = r.listen(source)

		return(r.recognize_google(audio))

if __name__ == "__main__":
	app = App()					# instantiate the app, client, counselor, and convo

	app.conversation.start()	# start the conversation


#Problem link:https://www.hackerrank.com/challenges/divisible-sum-pairs/problem
#Problem name:Divisible Sum Pairs
#Status:Easy
#Solution
def divisibleSumPairs(n, k, ar):
    Count=0
    for i in range(n):
        for j in range(i+1,n):
            if (ar[i]+ar[j])%k==0:
                Count+=1
    return Count


import xml.etree.ElementTree as ET, urllib, gzip, io
from system import System

url = "https://github.com/OpenExoplanetCatalogue/oec_gzip/raw/master/systems.xml.gz"
oec = ET.parse(gzip.GzipFile(fileobj=io.BytesIO(urllib.urlopen(url).read())))

system_objs = []

for system_node in oec.findall('.//system'):
    system_objs.append(System().build(system_node))

print (system_objs)


import PIL
from PIL import ImageFont
from PIL import Image
from PIL import ImageDraw

import os
import cv2
import numpy as np
import logging

# From  https://stackoverflow.com/questions/32504246/draw-ellipse-in-python-pil-with-line-thickness
def draw_circle(image, bounds, width=1, outline='white', antialias=4):
    """Improved ellipse drawing function, based on PIL.ImageDraw."""
    mask = Image.new(
        size=[int(dim * antialias) for dim in image.size],
        mode='L', color='black')
    draw = ImageDraw.Draw(mask)
    for offset, fill in (width/-2.0, 'white'), (width/2.0, 'black'):
        left, top = [(value + offset) * antialias for value in bounds[:2]]
        right, bottom = [(value - offset) * antialias for value in bounds[2:]]
        draw.ellipse([left, top, right, bottom], fill=fill)
    mask = mask.resize(image.size, Image.LANCZOS)
    image.paste(outline, mask=mask)

def draw_letter(letter):
    font = ImageFont.truetype("unicode.ttf", 75, encoding="unic")
    image=Image.new("RGBA", (200,200),(255,255,255,0))
    draw = ImageDraw.Draw(image)
    draw.text((70,45),letter,(0,0,0),font=font)
    draw = ImageDraw.Draw(image)
    image.save('temp.png')
    return image

def crop_circle(image):
    temp_img = cv2.imread('temp.png', 0)
    os.remove('temp.png')
    ret,thresh = cv2.threshold(temp_img,0,100,cv2.THRESH_BINARY_INV)
    _,contours,hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
    cnt = contours[0]

    ### With Moments
    # M = cv2.moments(cnt)
    # cx = int(M['m10']/M['m00'])
    # cy = int(M['m01']/M['m00'])

    ### With Center
    x,y,w,h = cv2.boundingRect(cnt)
    cx = int(x+w/2)
    cy = int(y+h/2)
    radius = 45
    width=5
    padding = 10
    circle = [int(cx)-radius,int(cy)-radius,int(cx)+radius,int(cy)+radius]
    draw_circle(image,circle, outline =(0,0,0),width=width)
    bound = (int(cx)-radius-width-padding,int(cy)-radius-width-padding,int(cx)+radius+width+padding,int(cy)+radius+width+padding)
    return image.crop(bound)

def setColor(im,R=255,G=255,B=255):
    data = np.array(im)
    # Clear, make all black pixel
    for i,alpha in enumerate(data[:,:,3]) :
        for j,v in enumerate(alpha) :
            if v != 0 :
                data[:,:,0],  data[:,:,1], data[:,:,2] =  0, 0, 0
    red, green, blue = data[:,:,0], data[:,:,1], data[:,:,2]
    mask = (red == 0) & (green == 0) & (blue == 0)
    data[:,:,:3][mask] = [R, G, B]
    return Image.fromarray(data)

def draw_word_in_circle(letter,path=''):
    letter = str(letter)
    file_name = letter + '.png'
    image = draw_letter(letter)
    image = crop_circle(image)
    image = setColor(image,190,190,190)
    image.save(path + file_name)

# Usage
for c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890+-' :
    try :
        draw_word_in_circle(c,path='./icon/')
    except :
        logging.warning('Directory not found, try to build \"icon\"')
        os.mkdir('icon')
        draw_word_in_circle(c,path='./icon/')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Python libraries
import copy
import sys
import pandas as pd
import ipdb

# Local imports

class LabelProcessor(object):

    """
    LabelProcessor is the class providing processing facilities to manage the
    labeled dataset

    The class provides facilities to:

        - Check if the category structure is a tree.
        - Give the appropriate format to label variables
        - Transfer new labels and preds to the whole data structure
        - Infer labels based on the tree category structure.
    """

    def __init__(self, categories, parentcat, log,
                 alphabet={'yes': 1, 'no': -1, 'unknown': 0, 'error': -99},
                 cat_model='single'):

        """
        States the alphabet of the Labels

        Args:
            :categories: Set of categories
            :parentcat:  Dictionary defining the tree category structure
            :log:        Logger
            :alphabet:   Possible labels for each category
            :cat_model:  Category model (monolabel or multilabel)
        """

        self.categories = categories
        self.parentcat = parentcat
        self.log = log
        self._yes = alphabet['yes']
        self._no = alphabet['no']
        self._unknown = alphabet['unknown']
        self._error = alphabet['error']
        self.cat_model = cat_model
        self.alphabet = alphabet

        # Check consistency of the label processing
        if not self.isTree():
            sys.exit("The structure of parent categories is not a tree")

    def isTree(self):

        """ Verifies it the graph structure defined by the parentcat dictionary
            is a tree and has no loops

            :Args:
                :-: The dictionary of categories is an object attribute
                            parentcat[c] is the parent of category c.
            :Returns:
                :isTree:    True is the graph is a Tree.
                            False if the graph has any loop.

            .. note:: For large trees, this code can be made much more
                      efficient by removing checked paths from the graph before
                      entering the next outer-loop step.
        """

        cats = self.parentcat.keys()
        isTree = True

        # Start an ascending path for every category in cats
        for c in cats:

            # The last visied node
            cold = c

            # A record of visited nodes
            FlagCats = dict.fromkeys(cats, 0)

            while isTree:

                # Register the last visited node
                FlagCats[cold] = 1

                # Move one step upwards
                cnew = self.parentcat[cold]

                if cnew not in cats:
                    # cnew is a root node. No loops here.
                    break
                else:
                    if FlagCats[cnew] == 1:
                        isTree = False
                        break
                cold = cnew

            if not isTree:
                break

        return isTree

    def formatData(self, df_labels):

        """ This method replaces any None or any unrecognized values
            in the labels from df_labels by an 'unknown' label.
            Also, it creates metadata columns if they do not exist.
        """

        if len(df_labels) > 0:
            df_labels.loc[df_labels['label'] not in self.alphabet,
                          self.categories] = self._unknown

            if ('info', 'marker') not in df_labels.columns:
                df_labels['info', 'marker'] = None
            if ('info', 'relabel') not in df_labels.columns:
                df_labels['info', 'relabel'] = None
            if ('info', 'weight') not in df_labels.columns:
                df_labels['info', 'weight'] = None
            if ('info', 'userId') not in df_labels.columns:
                df_labels['info', 'userId'] = None
            if ('info', 'date') not in df_labels.columns:
                df_labels['info', 'date'] = None

        return df_labels

    def cleanLabels(self, df_labels):

        """ Removes non recognized label values in the labels dictionary
            by the "unknown_label" codeword.

            WARNING: This method is likely obsolete, because the label cleaning
            is already being done by formatData.
        """

        if len(df_labels) > 0:
            df_labels.loc[df_labels['label'] not in self.alphabet,
                          self.categories] = self._unknown

        return df_labels

    def transferData(self, data2, data, mode='project'):

        """ Transfer predictions and labels to the pool of data managed by the
            DataMgr object.

            :Inputs:
                :data2: data to transfer.
                :data: destination dataset
                :mode: Type of transfer:
                    - 'project' (default): Only wids already existing in data
                                are taken from data2.
                    - 'expand': data is expanded with the new wids
                                in data2.

            :Returns:
                :-: No variables are returned. The methods modifies input
                    mutable object 'data'.

            WARNING: THIS METHOD IS DEPRECATED. IT HAS BEEN REPLACES BY
                     transferLabels() and transferPreds()
        """

        # Check consistency and infer labels from the tree structure

        # Data fusion
        for wid in data2:

            if wid in data:

                # Make sure that the label information is complete
                data2[wid]['label'] = self.inferLabels(data2[wid]['label'])

                # Only non "None" predictions are transferred
                for cat in self.categories:
                    if data2[wid]['pred'][cat] is not None:
                        data[wid]['pred'][cat] = data2[wid]['pred'][cat]

                # Transfer labels.
                label = data[wid]['label']
                label2 = data2[wid]['label']
                data[wid]['label'] = self.updateLabels(label2, label)

                # Transfer marker and relabel. Note that if there are no
                # marker or relabel variables in the new data, the original
                # value is preserved.
                if 'marker' in data2[wid]:
                    data[wid]['marker'] = data2[wid]['marker']

                if 'relabel' in data2[wid]:
                    data[wid]['relabel'] = data2[wid]['relabel']

                # Transfer weight. It is not clear if weights should be
                # transferred. Probably not, but probably the original value is
                # no longer valid ...
                if 'weight' in data2[wid]:
                    data[wid]['weight'] = data2[wid]['weight']

                # Transfer the identity of the human labelers
                if 'userId' in data2[wid]:
                    data[wid]['userId'] = data2[wid]['userId']

            elif mode == 'expand':

                data[wid] = {}

                # Transfer url
                data[wid]['url'] = data2[wid]['url']

                # Make sure that the label information is complete
                data2[wid]['label'] = self.inferLabels(data2[wid]['label'])

                # Transfer labels.
                data[wid]['label'] = data2[wid]['label']

                # Transfer predictions
                data[wid]['pred'] = data2[wid]['pred']

                # Transfer the rest of dictionary entries, if they exist
                if 'marker' in data2[wid]:
                    data[wid]['marker'] = data2[wid]['marker']
                else:
                    data[wid]['marker'] = None

                if 'relabel' in data2[wid]:
                    data[wid]['relabel'] = data2[wid]['relabel']
                else:
                    data[wid]['relabel'] = None

                if 'weight' in data2[wid]:
                    data[wid]['weight'] = data2[wid]['weight']
                else:
                    data[wid]['weight'] = None

                if 'userId' in data2[wid]:
                    data[wid]['userId'] = data2[wid]['userId']
                else:
                    data[wid]['userId'] = None

    def transferPreds(self, df2_preds, df_preds, mode='project'):

        """ Transfer predictions to the prediction dataframe managed by the
            DataMgr object.

            :Inputs:
                :df2_preds: data to transfer.
                :df_preds: destination dataset
                :mode: Type of transfer:
                    - 'project' (default): Only wids already existing in
                                df_preds are taken from df_preds2.
                    - 'expand': df_preds is expanded with the new wids
                                in df_preds2.
                    - 'contract': Wids that do not exist in df_preds2 are
                                removed

            :Returns:
                :df_preds:
        """

        # Remove useless rows.
        # All urls in df_preds of wids in df2_preds are copied into df2_preds
        # Wids in df2_preds that remain without url a removed

        # Original wid sets and their intersection
        widset = set(df_preds.index.tolist())
        widset2 = set(df2_preds.index.tolist())
        w_common = list(widset & widset2)

        # Copy urls from df_preds into df2_preds
        df2_preds.loc[w_common, 'url'] = df_preds.loc[w_common, 'url']

        # Remove predictions about wids whose url is unknown
        df2_preds = df2_preds[~df2_preds['url'].isnull()]

        # Transfer predicts
        if mode == 'contract':

            if len(df2_preds) == 0:
                exit("Transfer mode = contract is not permitted if " +
                     "df2_preds does not contain urls")

            # Remove the instances from df_preds that are not in df2_preds
            df_preds = df_preds[df_preds.index.isin(df2_preds.index)]

        if mode == 'contract' or mode == 'expand':

            # Add to df_preds the rows in df2_preds that are not in df_preds.
            df_preds = pd.concat(
                [df_preds, df2_preds[~df2_preds.index.isin(df_preds.index)]])

        # Update the rows in df_preds with the new data in df2_preds
        df_preds.update(df2_preds)

        return df_preds

    def transferLabels(self, df2_labels, df_labels):

        """ Transfer labels to the label dataframe managed by the DataMgr
            object.

            :Inputs:
                :df2_labels: data to transfer.
                :df_labels: destination dataset

            :Returns:
                :df_labels:
        """

        # Check consistency and infer labels from the tree structure

        # Data fusion
        for w in df2_labels.index:

            if w in df_labels.index:

                # Make sure that the label information is complete
                newlabel = df2_labels.loc[w].to_dict()

                df2_labels.loc[w]['label'] = self.inferLabels(newlabel)

                # Transfer labels.
                label = df_labels.loc[w]['label'].to_dict()
                label2 = df2_labels.loc[w]['label'].to_dict()
                df_labels.loc[w]['label'] = self.updateLabels(label2, label)

                # Transfer marker and relabel. Note that if there are no
                # marker or relabel variables in the new data, the original
                # value is preserved.
                if 'marker' in df2_labels.loc[w]['info']:
                    df_labels.loc[w, ('info', 'marker')] = df2_labels.loc[w, (
                        'info', 'marker')]

                if 'relabel' in df2_labels.loc[w]['info']:
                    df_labels.loc[w, ('info', 'relabel')] = df2_labels.loc[w, (
                        'info', 'relabel')]

                # Transfer weight. It is not clear if weights should be
                # transferred. Probably not, but probably the original value is
                # no longer valid ...
                if 'weight' in df2_labels.loc[w]['info']:
                    df_labels.loc[w, ('info', 'weight')] = df2_labels.loc[w, (
                        'info', 'weight')]

                # Transfer the identity of the human labelers
                if 'userId' in df2_labels.loc[w]['info']:
                    df_labels.loc[w, ('info', 'userId')] = df2_labels.loc[w, (
                        'info', 'userId')]

                # Transfer the identity of the human labelers
                if 'date' in df2_labels.loc[w]['info']:
                    df_labels.loc[w, ('info', 'date')] = df2_labels.loc[w, (
                        'info', 'date')]

            else:

                # All wids in df2_labels that are not in df_labels are added to
                # df_labels

                # Infer labels for all categories in the new label set
                newlabel = df2_labels.loc[w].to_dict()
                df2_labels.loc[w]['label'] = self.inferLabels(newlabel)

                # Transfer labels and info to df_labels
                df_labels.loc[w] = df2_labels.loc[w]

        return df_labels

    def transferNewLabels2(self, newlabels, df_labels, new_userId=None):

        """ Transfer a new set of labels contained in the labeling record.

            It assumes precise labeling: the category assigned to a wid is the
            finest subcategory containing the wid. Thus, the new category and
            all its ancestors are marked with the positive label, and any other
            category is marked with the negative label

            :Args:
                :newlabels: Dictionary of pairs wid: cat, where cat is the
                            new category assigned to the wid.
                :df_labels: The current label dataframe
                :new_userId: Name of the current human labeler

            :Returns:
                :df_labels:
        """

        for wid in newlabels:

            # New category
            cat = newlabels[wid]['label']

            # Only labels in the category set are transferred to the data
            # structure. This means that label "error" is ignored.
            # In case of "error", the pre-existing label information (i.e.
            # label value, marker and relabel) about the corresponding wid is
            # not modified.
            # [Despite of this, error events will be recorded in history files
            # (see transferLabelRecords())]
            if cat in self.categories:

                if wid not in df_labels:
                    if ('info', 'url') in df_labels.columns:
                        df_labels.loc[wid, ('info', 'url')] = (
                            newlabels[wid]['url'])

                # Insert the category in a label vector
                label_dict = dict.fromkeys(self.categories, self._unknown)
                label_dict[cat] = self._yes

                # Propagate the positive label to all ancestors. Also, a
                # negative label to the other non-descendant categories
                label_dict = self.inferLabels(label_dict)

                # Up to this point, the descendants of the given category have
                # not been modified. Since labeling is precise, they should
                # receive a negative label.
                # The following code assigns the negative class to all non
                # positive classes. This may be inefficient for large trees,
                # because some categories have been already labeled with the
                # negative class in the above lines.
                for c in self.categories:
                    if label_dict[c] != self._yes:
                        label_dict[c] = self._no

                for c in self.categories:
                    df_labels.loc[wid, ('label', c)] = label_dict[c]

                # Old "soft labeling". Insert new label in the data structure.
                # data[wid]['label'] = self.updateLabels(label_dict,
                #                                        label_orig)

                # Store the marker and relabel
                df_labels.loc[wid, ('info', 'marker')] = newlabels[wid][
                    'marker']
                df_labels.loc[wid, ('info', 'relabel')] = newlabels[wid][
                    'relabel']
                df_labels.loc[wid, ('info', 'date')] = newlabels[wid]['date']
                df_labels.loc[wid, ('info', 'userId')] = new_userId

        return df_labels

    def transferNewLabels(self, newlabels, data, new_userId=None):

        """ Transfer a new set of labels contained in the labeling record.

            It assumes precise labeling: the category assigned to a wid is the
            finest subcategory containing the wid. Thus, the new category and
            all its ancestors are marked with the positive label, and any other
            category is marked with the negative label

            :Args:
                :newlabels: Dictionary of pairs wid: cat, where cat is the
                            new category assigned to the wid.
                :data: The current data dictionary

            :Returns:
                :-: No variables are returned. The method modifies input
                    mutable object 'data'.
        """

        for wid in newlabels:

            # New category
            cat = newlabels[wid]['label']

            # Only labels in the category set are transferred to the data
            # structure. This means that label "error" is ignored.
            # In case of "error", the pre-existing label information (i.e.
            # label value, marker and relabel) about the corresponding wid is
            # not modified.
            # [Despite of this, error events will be recorded in history files
            # (see transferLabelRecords())]
            if cat in self.categories:

                if wid not in data:
                    data[wid] = {}
                    data[wid]['url'] = newlabels[wid]['url']

                # Insert the category in a label vector
                label_dict = dict.fromkeys(self.categories, self._unknown)
                label_dict[cat] = self._yes

                # Propagate the positive label to all ancestors. Also, a
                # negative label to the other non-descendant categories
                label_dict = self.inferLabels(label_dict)

                # Up to this point, the descendants of the given category have
                # not been modified. Since labeling is precise, they should
                # receive a negative label.
                # The following code assigns the negative class to all non
                # positive classes. This may be inefficient for large trees,
                # because some categories have been already labeled with the
                # negative class in the above lines.
                for c in self.categories:
                    if label_dict[c] != self._yes:
                        label_dict[c] = self._no
                data[wid]['label'] = label_dict

                # Old "soft labeling". Insert new label in the data structure.
                # data[wid]['label'] = self.updateLabels(label_dict,
                #                                        label_orig)

                # Store the marker and relabel
                data[wid]['marker'] = newlabels[wid]['marker']
                data[wid]['relabel'] = newlabels[wid]['relabel']
                data[wid]['date'] = newlabels[wid]['date']
                data[wid]['userId'] = new_userId

    def transferNewWeights2(self, newweights, df_labels):

        """ Transfer a weight dictionary to the dataset.

            :Args:
                :newweights: Dictionary of pairs wid: weight.
                :df_labels: The current label dataframe
                            Only the wids that are both in df_labels and
                            newweights are processed

            :Returns:
                :df_labels:
        """

        for wid in newweights:
            if wid in df_labels.index:
                df_labels.loc[wid, ('info', 'weight')] = newweights[wid]

        return df_labels

    def transferNewWeights(self, newweights, data):

        """ Transfer a weight dictionary to the dataset.

            :Args:
                :newweights: Dictionary of pairs wid: weight.
                :data: The current data dictionary.
                       Only the wids that are both in data and newweights are
                       processed

            :Returns:
                :-: No variables are returned. The method modifies input
                    mutable object 'data'.
        """

        for wid in newweights:
            if wid in data:
                data[wid]['weight'] = newweights[wid]

    def transferLabelRecords(self, newlabels, labelhistory, new_userId=None):

        """ Transfer a new set of labeling records contained in newlabels
            to the current data set.
        """

        for wid in newlabels:

            t = newlabels[wid]['date']
            tid = t.strftime("%Y%m%d%H%M%S%f")

            if wid not in labelhistory:
                labelhistory[wid] = {tid: newlabels[wid]}
            else:
                labelhistory[wid][tid] = newlabels[wid]

            # Add user Id.
            labelhistory[wid][tid]['userId'] = new_userId

    def updateLabels(self, label2, label):

        """ Updates label dictionary with the information provided by label2

            - If at least one category in label2 is ERROR, all categories are
              set to UNKNOWN
            - Any YES label in label2 is propagated through parent categories.
              Siblins of a node from category YES are set to zero.
              (i.e., sibling categories are assumed to be mutually exclusive)
            - Any NO label in label2 is propagated through child categories

            If the resulting label dictionary is not consistent (e.g. if all
            label components are NO), all categories are marked as UNKNOWN
        """

        # If thereis and "ERROR" label mark, all labels are removed.
        if any(label2[c] == self._error for c in self.categories):
            new_label = dict.fromkeys(label2, self._unknown)
        else:

            new_label = copy.copy(label2)

            # Preserve the old labels only if the new label is unknown
            for cat in self.categories:
                if new_label[cat] == self._unknown:
                    new_label[cat] = label[cat]

        label_out = self.inferLabels(new_label)

        if all(label_out[c] == self._unknown for c in self.categories):
            # self.inferLabels may have returned an 'all unknown' label
            # dictionary because all the input labels where 'no_label' (which
            # is inconsistent with the fact that all categories in the first
            # level of the tree are exhaustive).
            # For this reason, new_label is replaced by a copy of label2.
            label_out = copy.copy(label2)

        return label_out

    def inferLabels(self, label):

        """ Infer values of some label components based on the information
            of the observed labels and the category tree structure.
        """

        if any(label[c] == self._error for c in self.categories):
            # Any error in a label component invalidate the label
            # information
            new_label = dict.fromkeys(label, self._unknown)

        else:

            new_label = copy.copy(label)

            # Propagate labels
            for cat in self.categories:

                if new_label[cat] == self._yes:
                    new_label = self.upwards(new_label, cat)[0]
                elif new_label[cat] == self._no:
                    new_label = self.downwards(new_label, cat)[0]

        if all(label[c] == self._no for c in self.categories):
            # The category set is assumed to be exhaustive.
            # Therefore, if all values are NO, the labeling is not
            # consistent, and the label information must be removed
            new_label = dict.fromkeys(label, self._unknown)

        return new_label

    def upwards(self, label, cat):

        """ Propagates a label value "yes" up through the tree structure
        """

        isOK = True
        pcat = self.parentcat[cat]
        new_label = copy.copy(label)

        # Go up through the tree
        if pcat in self.categories:

            if new_label[pcat] == self._no:
                # Inconsistency. Return unknown to relabel
                isOK = False
                new_label = dict.fromkeys(new_label, self.unknown)
            elif new_label[pcat] == self._unknown:
                # Recursive call
                new_label[pcat] = self._yes
                new_label, isOK = self.upwards(new_label, pcat)

        if isOK:
            # Propagate a "NO" through the sibling categories
            for c in self.findSiblings(new_label, cat):
                if new_label[c] == self._yes:
                    # Labeling is inconsistent, because we are assuming
                    # that sibling categories are mutually exclusive
                    isOK = False
                    new_label = dict.fromkeys(new_label, self._unknown)
                    break
                else:
                    new_label[c] = self._no
                    new_label, isOK = self.downwards(new_label, c)
                    if not isOK:
                        break

        return new_label, isOK

    def downwards(self, label, cat):

        """ Propagates a label value "no" down through the tree structure
        """

        isOK = True
        new_label = copy.copy(label)

        for c in self.categories:

            pc = self.parentcat[c]

            # If c is a child of cat...
            if pc == cat:
                if new_label[c] == self._yes:
                    # Inconsistency. Return unknown to relabel
                    isOK = False
                    new_label = dict.fromkeys(new_label, self._unknown)
                    break
                elif new_label[c] == self._unknown:
                    new_label[c] = self._no
                    new_label, isOK = self.downwards(new_label, c)
                    if not isOK:
                        break

        return new_label, isOK

    def findSiblings(self, label, cat):

        """ Finds the siblins of node 'cat' in the label tree structure
            (two nodes are siblings if they share the same parent)
        """

        pc = self.parentcat[cat]
        sib = [c for c in self.categories
               if self.parentcat[c] == pc and c != cat]

        return sib


from flask_restful import Resource, reqparse
from flask_jwt import jwt_required
from models.color import ColorModel

class Color(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument(
        'rgbvalue',
        type = str,
        required = True,
        help = 'This value should be of type String e.g. R:0 G:0 B:0'
    )

    @jwt_required()
    def get(self, user_id, id):
        color = ColorModel.find_by_id(id)

        if color is None:
            return {
                'message': 'Color does not exist.'
            }, 404

        elif color.user_id == user_id:
            return color.json()

        return {
            'message': 'Not enough permissions to process the request.'
        }, 400

    @jwt_required()
    def post(self, user_id):
        body = Color.parser.parse_args()
        color = ColorModel.find_by_value(body['rgbvalue'])
        if color:
            return color.json()

        color = ColorModel(body['rgbvalue'], user_id)
        try:
            color.save_to_db()
            return color.json()
        except:
            return {
                'message': 'An error occured inserting the color.'
            }, 500

    @jwt_required()
    def delete(self, user_id, id):
        color = ColorModel.find_by_id(id)

        if color and color.user_id == user_id:
            color.delete_from_db()
            return {
                'message': 'Color removed!'
            }

        return {
            'message': 'Not enough permissions to process the request.'
        }, 400

class ColorList(Resource):
    @jwt_required()
    def get(self, user_id):
        return ColorModel.find_all_by_user_id(user_id)


import warnings
warnings.filterwarnings("ignore")

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

# %%
def obj_df(df):
    """
    Returns the dataframe only containing object columns
    Paramter: df
    """
    mask = np.array(df.dtypes == 'object')
    df_obj = df.iloc[:, mask]
    return df_obj

# %%
def sum_obj_cols (df):
    """
    Returns the object columns with their unique values (don't include NaN in counts).
    Paramter: df
    """
    df_obj = obj_df(df)
    obj_cols = pd.DataFrame(df_obj.dtypes, columns=['dtypes'])
    obj_cols['unique_values'] = df_obj.nunique()
    return obj_cols

# %%
def obj_value_counts(df):
    """
    Count the unique values of each object column in the dataframe.
    Parameter: df
    """
    df_obj = obj_df(df)
    for col in df_obj.columns:
        print(df_obj[col].value_counts())
        print('-'*100)

# %%
def num_df(df):
    """
    Subset the dataframe only containing float or int columns
    Paramter: df
    """
    float_mask = np.array(df.dtypes == 'float')
    int_mask = np.array(df.dtypes == 'int')
    mask = float_mask | int_mask
    df_obj = df.iloc[:, mask]
    return df_obj

# %%
def plot_variable_pairs(df):
    """
    Pair-plot the variables
    Parameter: df
    """
    return sns.pairplot(df, kind='reg')

# %%
def sum_missing_values_attributes(df):
    """
    Count how many missing values in each attribute
    Parameter: interested df
    """
    missing_values = pd.DataFrame(df.isna().sum(axis=0), columns=['num_row_missing'])
    total_rows = df.shape[0]
    missing_values['pct_rows_missing'] = missing_values.num_row_missing/total_rows
    return missing_values

# %%
def sum_missing_values_cols(df):
    """
    Group the rows based on how many missing values they have
    Parameter: interested df
    """
    missing_values = df.isnull().sum(axis=1).value_counts().sort_index()
    cols_missing_values = {'num_cols_missing': missing_values.index.tolist(), 'num_rows': missing_values.values.tolist()}
    cols_missing_values = pd.DataFrame(cols_missing_values)
    total_rows = df.shape[0]
    cols_missing_values['pct_cols_missing'] = (cols_missing_values.num_rows/total_rows)*100
    return cols_missing_values


import tensorflow as tf

import os
import subprocess

flags = tf.app.flags

FLAGS = flags.FLAGS

flags.DEFINE_string('input_dir', '/home/ace19/dl_data/ModelNet40/xbox',
                    'input dir')

flags.DEFINE_string('output_dir', '/home/ace19/dl_data/ModelNet40/xbox',
                    'output dir')

flags.DEFINE_string('output_file_ext', '.obj',
                    'output file extension')

flags.DEFINE_string('sub_dir', 'test',
                    'train or test')

def main(unused_argv):
    input = os.path.join(FLAGS.input_dir, FLAGS.sub_dir)
    off_files = os.listdir(input)
    off_files.sort()

    for file in off_files:
        file_name = os.path.basename(file)[:-4]
        output_file_path = os.path.join(FLAGS.output_dir, FLAGS.sub_dir, file_name + FLAGS.output_file_ext)
        cmd = 'ctmconv ' + \
              os.path.join(input, file) + ' ' \
              + output_file_path
        os.system(cmd)
        # subprocess.call(cmd, shell=True)

if __name__ == '__main__':
    tf.app.run()


from pygame import *

screen = display.set_mode((980, 750))
display.set_caption ("Paint")

running = True
while running:
    for evt in event.get():
        if evt.type == QUIT:
            running = False

        if evt.type == MOUSEBUTTONDOWN:
            back = screen.copy()
            if evt.button == 1 or evt.button == 3:
                omx, omy = evt.pos       #old mx, old my - used for drawing and dragging lines
    mb = mouse.get_pressed()
    mx,my = mouse.get_pos()
    if mb[0] == 1:
        screen.blit(back,(0,0))
        draw.line(screen,(255,0,0),(omx,omy), (mx,my))

    display.flip()
quit()


import numpy as np
import math

def SLR(Y,res):
    #print(n)
    row=len(Y)
    temp=np.ones(row)
    #print(np.shape(temp))
    #print(temp)
    res=np.abs(res)
    #print(res)
    res=np.vstack([temp,res])
    #print(np.shape(res))
    res_transpose=np.transpose(res)
    #print(np.shape(res_transpose))
    temp=np.matmul(res,res_transpose)
    #print(np.shape(temp))
    temp=np.linalg.inv(temp)
    temp=np.matmul(temp,res)
    LSE=np.matmul(temp,Y)
    fitted_Y=np.matmul(res_transpose,LSE)
    #print(fitted_Y)
    return fitted_Y

def weight_cal(Y,res):

    fitted_Y=SLR(Y,res)
    row = len(fitted_Y)
    weighted_value = np.identity(row)
    for i in range(0,row):
        for j in range(0,row):
            if(j==i):
                weighted_value[i][j]=(1/(math.log10(fitted_Y[i])))
    return weighted_value


from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.wait import WebDriverWait

class TestWx():
    def setup(self):
        chromeOptions=Options()
        # 导入options
        # 添加参数
        chromeOptions.add_experimental_option("debuggerAddress","127.0.0.1:9222")
        self.driver = webdriver.Chrome(options=chromeOptions)
        # chrome --remote-debugging-port=9222 打开新的窗口
        self.driver.implicitly_wait(3)

    def test_tianjia(self):
        # self.driver.find_element(By.CSS_SELECTOR, ".index_service_cnt_itemWrap div:nth-child(1)").click()
        # 首页直接点击添加成员
        self.driver.find_element(By.ID, "menu_contacts").click()
        sleep(10)
        # 首页点击通讯录，添加显示等待
        element=(By.CSS_SELECTOR, ".js_has_member div:nth-child(1) .js_add_member")
        WebDriverWait(self.driver, 10).until(expected_conditions.element_to_be_clickable(element))
        self.driver.find_element(*element).click()
        self.driver.find_element(By.ID, "username").send_keys("abcd")
        self.driver.find_element(By.ID, "memberAdd_acctid").send_keys("1234567")
        self.driver.find_element(By.ID, "memberAdd_phone").send_keys("13512345677")
        self.driver.find_element(By.CSS_SELECTOR, ".js_btn_save").click()

    def teardown(self):
        sleep(5)
        self.driver.quit()


#!/usr/bin/env python

from distutils.core import setup

## Need to import the CFFI module during installation to ensure it
## builds the packages, and places them in the right context.
try:
      import petlib.bindings
      deps = [petlib.bindings._FFI.verifier.get_extension()]
except Exception as e:
      import traceback
      traceback.print_exc()
      print("Alter: Not compiling the library -- useful for readthedocs.")
      deps = []

import petlib

setup(name='petlib',
      version=petlib.VERSION,
      description='A library implementing a number of Privacy Enhancing Technologies (PETs)',
      author='George Danezis',
      author_email='g.danezis@ucl.ac.uk',
      url=r'https://pypi.python.org/pypi/petlib/',
      packages=['petlib'],
      ext_package='petlib',
      license="2-clause BSD",
      long_description="""A library wrapping Open SSL low-level cryptographic libraries to build Privacy Enhancing Technoloies (PETs)""",
      install_requires=[
            "cffi >= 0.8.2",
            "future >= 0.14.3",
            "pytest >= 2.6.4",
            "paver >= 1.2.3",
            "pytest-cov >= 1.8.1",
      ],
      zip_safe=False,
      ext_modules=deps,
)


import os
import logging
import argparse
from pkg_resources import resource_filename
from functools import partial
from shutil import copytree, rmtree, copy
from pds_github_util.branches.git_actions import loop_checkout_on_branch
from pds_github_util.gh_pages.summary import write_build_summary
from pds_github_util.gh_pages.root_index import update_index

logger = logging.getLogger('github3')
logger.setLevel(level=logging.WARNING)

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()

def copy_resources():
    logger.info("write static resources (img, config)...")
    resources = resource_filename(__name__, 'resources')
    for f in os.listdir(resources):
        i_p = os.path.join(resources, f)
        o_p = os.path.join(os.getcwd(), f)
        if os.path.isdir(i_p):
            if os.path.exists(o_p):
                rmtree(o_p, ignore_errors=True)
            copytree(i_p, o_p)
        else:
            if os.path.exists(o_p):
                os.remove(o_p)
            copy(i_p, os.getcwd())

def build_summaries(token, path=os.getcwd(), format='md'):
    copy_resources()

    herds = []
    # loop on stable releases
    for herd in loop_checkout_on_branch('NASA-PDS/pdsen-corral',
                            '[0-9]+\.[0-9]+',
                            partial(write_build_summary,
                                    root_dir=path,
                                    gitmodules='/tmp/pdsen-corral/.gitmodules',
                                    token=token,
                                    dev=False,
                                    format=format),
                            token=token,
                            local_git_tmp_dir='/tmp'):
        herds.append(herd)

    # dev release on master
    herd = next(loop_checkout_on_branch('NASA-PDS/pdsen-corral',
                            'master',
                            partial(write_build_summary,
                                    root_dir=path,
                                    gitmodules='/tmp/pdsen-corral/.gitmodules',
                                    token=token,
                                    dev=True,
                                    format=format),
                            token=token,
                            local_git_tmp_dir='/tmp'))

    herds.append(herd)

    update_index(path, herds)

def main():
    parser = argparse.ArgumentParser(description='Create new snapshot release')
    parser.add_argument('--token', dest='token',
                        help='github personal access token')
    parser.add_argument('--path', dest='path',
                        help='directory where the summary will be created')
    parser.add_argument('--format', dest='format',
                        help='format of the summary, accepted formats are md and rst')
    args = parser.parse_args()
    build_summaries(args.token, args.path, args.format)

if __name__ == "__main__":
    main()