from flask import Flask
from flask_restx import Api, Resource, fields
from werkzeug.middleware.proxy_fix import ProxyFix

app = Flask(__name__)
app.config['RESTX_MASK_SWAGGER'] = False
app.wsgi_app = ProxyFix(app.wsgi_app)
api = Api(app, version="1.0.0", title="astroneer", description="An astroneer app", prefix='/astro/v1')

ns = api.namespace("astroneer", description="Astroneer operations")  # todo why have namespaces at all?

DATABASE = {'modules': [], 'resources': [], 'planets': []}

resource_model = api.model("Resource", {
    "name": fields.String(required=True, description="The name of the resource"),
    "found": fields.List(fields.String, required=True, description="Planets on which this resource is found")
})
resource_list = api.model("ResourceList", {
    "resources": fields.List(fields.Nested(resource_model, description="Resource"), description="Resources"),
})

module_model = api.model("Module", {
    "name": fields.String(required=True, description="The name of the module"),
    "resource_cost": fields.List(fields.String, description="Resource cost to craft the module in a printer")
})
module_list = api.model("ModuleList", {
    "modules": fields.List(fields.Nested(module_model, description="Module"), description="Modules"),
})

def abort_if_module(module, **kwargs):
    test = [x['name'] for x in DATABASE['modules'] if x['name'] == module]
    if 'not_exists' in kwargs:
        test = not test
        api.abort(404, "module {} doesn't exist".format(module))
    if test:  # if empty, 404
        api.abort(400, "Module {} already exists".format(module))

def abort_if_resource(resource, **kwargs):
    test = [x['name'] for x in DATABASE['resources'] if x['name'] == resource]
    if 'not_exists' in kwargs:
        test = not test
        api.abort(404, "resource {} doesn't exist".format(resource))
    if test:  # if empty, 404
        api.abort(400, "Resource {} already exists".format(resource))

@ns.route("/")
class Debug(Resource):

    def get(self):
        """Debug print"""
        return DATABASE

resource_parser = api.parser()
resource_parser.add_argument("name", type=str, required=True, help="Resource name", location="form")
resource_parser.add_argument("found", type=str, required=True, help="Planets on which this resource is found",
                             location="form")
module_parser = api.parser()
module_parser.add_argument("name", type=str, required=True, help="Resource name", location="form")
module_parser.add_argument("resource_cost", type=str, required=True, help="Resource cost to print the module",
                           location="form")

#
# resources
#

@ns.route("/resource/<string:name_id>")
@api.doc(responses={404: "Resource not found"}, params={"name_id": "The resource name"})
class ResourceApi(Resource):
    """Show a single todo item and lets you delete them"""

    @api.doc(description="Resource name should be in the database")
    @api.marshal_with(resource_model)
    def get(self, name_id):
        """Fetch a given resource"""
        abort_if_resource(name_id, not_exists=True)
        return [x for x in DATABASE['resources'] if x['name'] == name_id][0]

    @api.doc(responses={204: "Resource deleted"})
    def delete(self, name_id):
        """Delete a given resource"""

        abort_if_resource(name_id, not_exists=True)
        DATABASE['resources'].remove([x for x in DATABASE['resources'] if x['name'] == name_id][0])
        return "", 204

    @api.doc(parser=resource_parser)
    @api.marshal_with(resource_model)
    def put(self, name_id):
        """Update a given resource"""
        abort_if_resource(name_id, not_exists=True)
        args = resource_parser.parse_args()
        resource = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['resources'].remove([x for x in DATABASE['resources'] if x['name'] == name_id][0])
        DATABASE['resources'].append(resource)
        return resource

@ns.route("/resource/")
class ResourceListApi(Resource):
    """Shows a list of all resources, and lets you POST to add new resources"""

    @api.marshal_list_with(resource_list)
    def get(self):
        """List all resources"""
        return {'resources': DATABASE['resources']}

    @api.doc(parser=resource_parser, responses={400: "Resource already exists"})
    @api.marshal_with(resource_model, code=201)
    def post(self):
        """Create a resource"""
        args = resource_parser.parse_args()
        abort_if_resource(args["name"], exists=True)
        resource = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['resources'].append(resource)
        return resource, 201

#
# modules
#

@ns.route("/module/<string:name_id>")
@api.doc(responses={404: "Module not found"}, params={"name_id": "The module name"})
class ModuleApi(Resource):
    """Show a single todo item and lets you delete them"""

    @api.doc(description="Module name should be in the database")
    @api.marshal_with(module_model)
    def get(self, name_id):
        """Fetch a given module"""
        abort_if_module(name_id, not_exists=True)
        return [x for x in DATABASE['modules'] if x['name'] == name_id][0]

    @api.doc(responses={204: "Module deleted"})
    def delete(self, name_id):
        """Delete a given module"""

        abort_if_module(name_id, not_exists=True)
        DATABASE['modules'].remove([x for x in DATABASE['modules'] if x['name'] == name_id][0])
        return "", 204

    @api.doc(parser=module_parser)
    @api.marshal_with(module_model)
    def put(self, name_id):
        """Update a given module"""

        abort_if_module(name_id, not_exists=True)
        args = module_parser.parse_args()
        module = {
            "name": args["name"],
            "found": [x.strip() for x in args["found"].split(',')]
        }
        DATABASE['modules'].remove([x for x in DATABASE['modules'] if x['name'] == name_id][0])
        DATABASE['modules'].append(module)
        return module

@ns.route("/module/")
class ModuleListApi(Resource):
    """Shows a list of all modules, and lets you POST to add new modules"""

    @api.marshal_list_with(module_list)
    def get(self):
        """List all modules"""
        return {'modules': DATABASE['modules']}

    @api.doc(parser=module_parser, responses={400: "Module already exists"})
    @api.marshal_with(module_model, code=201)
    def post(self):
        """Create a module"""
        args = module_parser.parse_args()
        abort_if_module(args["name"], exists=True)
        module = {
            "name": args["name"],
            "resource_cost": [x.strip() for x in args["resource_cost"].split(',')]
        }
        DATABASE['modules'].append(module)
        return module, 201

if __name__ == "__main__":
    app.run(debug=True)


import torch
from torch import Tensor
import torch.nn.functional as F
from _utils_ import trg_mask

def first_prediction(image, encoder, decoder, conf, start=0):
    encoder.eval()
    decoder.eval()
    start_token = start
    feat = encoder(image)
    encoded, weights = decoder.encoder_att(feat=feat)

    outputs = Tensor([start_token]).long()

    trg_mask_ = trg_mask(outputs.unsqueeze(0))
    out = decoder.transformer(memory=encoded, trg=outputs.unsqueeze(0), trg_mask=trg_mask_)
    out = F.softmax(out, dim=-1)
    probs, ix = out[:, -1].data.topk(conf.k)
    log_scores = torch.log(probs)
    # log_scores = probs
    # print("log: ", log_scores)
    outputs = torch.zeros(conf.k, conf.max_dec_seq).long()
    outputs[:, 0] = start_token
    outputs[:, 1] = ix[0]
    encoded_outputs = torch.zeros(conf.k, encoded.size(-2), encoded.size(-1))
    encoded_outputs[:, :] = encoded[0]

    return outputs, encoded_outputs, log_scores, weights

def k_best_outputs(outputs, out, log_scores, i, k):
    probs, ix = out[:, -1].data.topk(k)
    # print("Top-K", out[:, :].data.topk(10))
    # E.g., out -> [[0.9, 0.8, 0.7], [0.7, 0.91, 0.89], [0.4, 0.98, 0.77]]
    log_probs = torch.log(probs) + log_scores
    # log_probs = probs + log_scores
    k_probs, k_ix = log_probs.view(-1).topk(k)
    # print("log prob: ", log_probs)
    row = k_ix // k
    col = k_ix % k
    # print("row, col: ", row, col)
    outputs[:, :i] = outputs[row, :i]
    outputs[:, i] = ix[row, col]

    log_scores = k_probs.unsqueeze(0)

    return outputs, log_scores

def beam_search(image, encoder, decoder, conf, end=1):
    outputs, en_outputs, log_scores, weights = first_prediction(image,
                                                       encoder,
                                                       decoder,
                                                       conf)
    end_token = end
    ind = None
    # print("Out::", outputs.size(), en_outputs.size(), log_scores.size())
    for i in range(2, conf.max_dec_seq):
        trg_mask_ = trg_mask(torch.LongTensor(i).unsqueeze(0))
        out = decoder.transformer(memory=en_outputs,
                                  trg=outputs[:, :i],
                                  trg_mask=trg_mask_)
        out = F.softmax(out, dim=-1)
        outputs, log_scores = k_best_outputs(outputs,
                                             out,
                                             log_scores,
                                             i,
                                             conf.k)
        none_end = torch.nonzero(outputs == end_token)
        sentence_lengths = torch.zeros(len(outputs)).long()

        for vec in none_end:
            i = vec[0]
            if sentence_lengths[i] == 0:
                sentence_lengths[i] = vec[1]

        num_finished_sentences = len([s for s in sentence_lengths if s > 0])
        if num_finished_sentences == conf.k:
            alpha = 0.7
            div = 1/(sentence_lengths.type_as(log_scores)**alpha)
            # print("div: ", log_scores)
            _, ind = torch.max(log_scores*div, 1)
            # print("Ind: ", ind)
            ind = ind.data[0]
            # ind = torch.argmax(log_scores, dim=-1).data[0]
            break

    if ind is None:
        length = torch.nonzero(outputs[0] == end_token)[0]
        sentence = [tok.item() for tok in outputs[0][1:length]]
        return sentence, weights
    else:
        length = torch.nonzero(outputs[ind] == end_token)[0]
        sentence = [tok.item() for tok in outputs[ind][1:length]]
        return sentence, weights


import csv
import os

from django.core.management import BaseCommand

from reviews.models import (
    User,
    Category,
    Review,
    Comment,
    Title,
    Genre,
    GenreTitle
)

FILES_DIR = 'static/data'

CLASSES = {
    'users.csv': User,
    'category.csv': Category,
    'titles.csv': Title,
    'review.csv': Review,
    'comments.csv': Comment,
    'genre.csv': Genre,
    'genre_title.csv': GenreTitle
}

FOREIGN_FIELDS = {
    'category': ('category', Category),
    'title_id': ('title', Title),
    'genre_id': ('genre', Genre),
    'author': ('author', User),
    'review_id': ('review', Review),
}

def open_file(file):
    file_path = os.path.join(FILES_DIR, file)
    with open(file_path, encoding='utf-8') as csv_file:
        return list(csv.reader(csv_file))

def load_file(file, model_type):
    data = open_file(file)
    rows = data[1:]
    for row in rows:
        csv_data = dict(zip(data[0], row))
        new_data_csv = dict(csv_data)
        for key, value in csv_data.items():
            if key in FOREIGN_FIELDS.keys():
                new_key = FOREIGN_FIELDS[key][0]
                new_data_csv[new_key] = (
                    FOREIGN_FIELDS[key][1].objects.get(pk=value)
                )
        database = model_type(**new_data_csv)
        database.save()

class Command(BaseCommand):
    """Загрузка данных, для тестирования."""

    help = 'Загрузка данных'

    def handle(self, *args, **options):
        for key, value in CLASSES.items():
            load_file(key, value)


from bson import ObjectId
from nose import with_setup

from mlight.attributes import FieldProperty
from mlight.meta_model import MetaModel
from tests.common import get_db_session, drop_all_collections, loop_runner, drop_database

db_session = get_db_session()

def setup_module():
    print("Module '%s' setup" % __name__)

def teardown_module():
    drop_all_collections(db_session=db_session)
    drop_database(db_session)
    print("Module '%s' teardown" % __name__)

def setup_function():
    db_session.clear_all()
    print("\nTest setup")

def teardown_function():
    db_session.clear_all()
    drop_all_collections(db_session=db_session)
    print("Test teardown")

class QueryDocument(MetaModel):
    session = db_session
    __model__ = 'indexed_document'

    _id = FieldProperty(ObjectId, if_missing=ObjectId)
    name = FieldProperty(str, required=True, if_missing='')
    age = FieldProperty(int, required=True)
    height = FieldProperty(float)

db_session.register_model(QueryDocument)

@with_setup(setup_function, teardown_function)
def test_query_document_get():
    obj = QueryDocument(age=1231231)

    async def run_async():
        await obj.flush()
        query_obj = await QueryDocument.get(obj._id)

        assert obj._id == query_obj._id, '_id field should match'
        assert obj.name == query_obj.name, 'name field should match'
        assert obj.age == query_obj.age, 'age field should match'
        assert obj.height == query_obj.height, 'height field should match'

    loop_runner(run_async)

@with_setup(setup_function, teardown_function)
def test_query_find():
    obj1 = QueryDocument(age=8716868)
    obj2 = QueryDocument(name='saassasa', age=12313, height=123.)

    def check_objects(obj, query_obj):
        assert obj._id == query_obj._id, '_id field should match'
        assert obj.name == query_obj.name, 'name field should match'
        assert obj.age == query_obj.age, 'age field should match'
        assert obj.height == query_obj.height, 'height field should match'

    async def run_async():
        await obj1.flush()
        await obj2.flush()

        results = await QueryDocument.find()

        result1 = results[0]
        result2 = results[1]

        if obj1._id == result1._id:
            check_objects(obj1, result1)
            check_objects(obj2, result2)
        else:
            check_objects(obj1, result2)
            check_objects(obj2, result1)

    loop_runner(run_async)

@with_setup(setup_function, teardown_function)
def test_query_collection_find():
    obj1 = QueryDocument(age=8716868)
    obj2 = QueryDocument(name='saassasa', age=12313, height=123.)

    def check_objects(obj, query_obj):
        assert obj._id == query_obj._id, '_id field should match'
        assert obj.name == query_obj.name, 'name field should match'
        assert obj.age == query_obj.age, 'age field should match'
        assert obj.height == query_obj.height, 'height field should match'

    async def run_async():
        await obj1.flush()
        await obj2.flush()

        cursor = QueryDocument.collection.find()
        results = await QueryDocument.to_mapped_list(cursor)

        result1 = results[0]
        result2 = results[1]

        if obj1._id == result1._id:
            check_objects(obj1, result1)
            check_objects(obj2, result2)
        else:
            check_objects(obj1, result2)
            check_objects(obj2, result1)

    loop_runner(run_async)

    # query a document update it and save it in the session!


# #重点:
# 导入PANDAS, 利用PANDAS 的数据绘制关系图

# 如何自行输入经纬度绘制所在区域的地图

# 学习如何读取气象资讯档: netCDF4

# 作业
# 读入 etopo5 地形/降水量资料集: "etopo5.cdf". 绘制分布图.
# 延续使用者资料的导入使用, 除了, "文字档", "网络资讯", 还有图像档;
# 所以尝试使用一张"earth_lights_lrg.jpg" 搭配不同的投影方式绘制地图

from mpl_toolkits.basemap import Basemap, shiftgrid, cm
import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset

# 读入 etopo5 地形/降水量。.
#url = 'https://github.com/NOAA-PMEL/FerretDatasets/blob/master/data/etopo5.cdf'
# 要确认资料集所在路径
etopodata = Dataset('etopo5.cdf')
# 从这里开始写code
# 取得资料集里的资料定义, 使用variables.keys()
etopodata .variables.keys()
# dict_keys(['ETOPO05_X', 'ETOPO05_Y', 'ROSE'])

# # 汇入资料 使用variables['keys'][:]
lons = etopodata.variables['ETOPO05_X'][:]
topoin = etopodata.variables['ROSE'][:]
lats =  etopodata.variables['ETOPO05_Y'][:]

# # 移位资料, 使 lon 从 - 180 到 180, 而不是 20 到 380
topoin, lons = shiftgrid(180.,topoin ,lons ,start=False)

# # 绘制地形/降水量
# # 创建图形和轴实例
fig = plt.figure()
ax = fig.add_axes([0.1,0.1,0.8,0.8])

# #设定地理资讯图('lcc' = lambert conformal conic).
# # 使用 Wgs84 椭圆体的主要和次要球体半径, 做基础的底图
# # 请填入 resolution='??',area_thresh=???.,projection='??'
# # area_thresh 为一数字, 低于这个数字不呈现

m = Basemap(llcrnrlon=-145.5,llcrnrlat=1.,urcrnrlon=-2.566,urcrnrlat=46.352,\
             rsphere=(6378137.00,6356752.3142),\
             resolution='l',area_thresh=10000.,projection='lcc',\
             lat_1=50.,lon_0=-107.,ax=ax )

# # 转换为 nx x ny 定期分空 5km 本机投影网格
nx = int((m.xmax-m.xmin)/5000.)+1 ; ny = int((m.ymax-m.ymin)/5000.)+1
topodat = m.transform_scalar(topoin,lons,lats,nx,ny)
# # plot image over map with imshow.
im = m.imshow(topodat,cm.GMT_haxby)
# #划定海岸线和边境界限, 总共需要3个参数
m.drawcountries()
m.drawmapboundary()   # 绘制边界
m.drawcoastlines()    # 绘制海岸线
m.drawstates()        # 绘制州

# # 绘制经纬线
# # 设定地图左侧和底部的标签
parallels = np.arange(-90.,120.,30.)
m.drawparallels(parallels, labels = [1,0,0,1])
meridians = np.arange(0.,420.,60.)
m.drawmeridians(meridians,labels = [1,0,0,1] )

# # 加入 colorbar
cb = m.colorbar(im,"right", size="5%", pad='2%')
ax.set_title('ETOPO5 Topography - Lambert Conformal Conic')
plt.show()

#from __future__ import (absolute_import, division, print_function)

from mpl_toolkits.basemap import Basemap
import numpy as np
import matplotlib.pyplot as plt

# 说明使用warpimage method 方法显示图像背景。
# 在地图投影区域。 预设背景是‘蓝色。
#来自美国宇航局的大理石图像 (http://visibleearth.nasa.gov)

# 建立新图像
fig=plt.figure()
# 定义以北美为中心的正交投影.
m = Basemap(projection='ortho',lat_0=40,lon_0=-100,resolution='l')
# 显示非预设影像 - 汇入要使用的IMAGE
m.warpimage(image='earth_lights_lrg.jpg')
# 绘制海岸线。
m.drawcoastlines(linewidth=0.5,color='0.5')
# 每 30 度绘制一组 lat/lon 网格线。
m.drawmeridians(np.arange(0,360,30),color='0.5')
m.drawparallels(np.arange(-90,90,30),color='0.5')
#添加图示标题
plt.title("Lights at Night image warped from 'cyl' to 'ortho' projection",fontsize=12)
plt.show()
print('warp to orthographic map ...')

#从这里开始写code

# 建立新图像
fig = plt.figure()

#新定义圆柱形等距投影。
m = Basemap(projection='cyl',llcrnrlon=-180,llcrnrlat=-90,urcrnrlon=180,urcrnrlat=90,resolution='l')

# 绘图 (未扭曲) rgba 图像
im = m.bluemarble(scale=0.5)
m.warpimage(image='earth_lights_lrg.jpg')
# 绘制海岸线。

# 绘制经纬度网格
m.drawmeridians(np.arange(-180,180,60),labels=[0,0,0,1],color='0.5')

parallels = np.arange(-90.,120.,30.)
m.drawparallels(parallels, labels = [1,0,0,1])

# 输出图像标题
plt.title("Blue Marble image - native 'cyl' projection",fontsize=12)
print('plot native cylindrical map (no warping needed) ...')

plt.show()


from __future__ import print_function, unicode_literals

import sys

reload(sys)
sys.setdefaultencoding('utf8')

import getopt
import json
import codecs
import os
import librosa

def read_audio_entries(data_file):
    json_data = json.load(codecs.open(data_file, 'r', 'utf-8'))
    audio_entries = list()
    for entry in json_data['fragments']:
        audio_entry = AudioEntry(entry)
        audio_entries.append(audio_entry)
    return audio_entries

def make_split(data_file, sound_file, start_shift_in_seconds=-0.1, end_shift_in_seconds=-0.1):
    audio_entries = read_audio_entries(data_file=data_file)
    result_path = os.path.splitext(sound_file)[0]
    if not os.path.exists(result_path):
        os.makedirs(result_path)

    return cut_sound_files_librosa(audio_entries, sound_file, result_path, start_shift_in_seconds, end_shift_in_seconds)

def cut_sound_files_librosa(audio_entries, sound_file, result_folder_path, start_shift_in_seconds=-0.1, end_shift_in_seconds=-0.1):
    audio_data, sample_rate = librosa.load(sound_file)
    for audio_entry in audio_entries:
        if len(audio_entry.text[0]) == 0:
            continue

        start = float(audio_entry.start)
        if start < 0:
            start = 0.0

        end = float(audio_entry.end)
        if end <= start:
            continue

        start_i = int(start * sample_rate)
        end_i = int(end * sample_rate)
        segment = audio_data[start_i:end_i]
        librosa.output.write_wav(os.path.join(result_folder_path, audio_entry.id) + '.wav', segment, sample_rate)
    return True

class AudioEntry:
    def __init__(self):
        self.start = None
        self.end = None
        self.id = None
        self.text = None

    def __init__(self, json_entry):
        self.start = json_entry['begin']
        self.end = json_entry['end']
        self.id = json_entry['id']
        self.text = json_entry['lines']

def print_usage():
    print('Usage:')
    print('\tpython audio_split.py -j <jsonfile> -a <audiofile>')
    sys.exit(1)

if __name__ == '__main__':
    try:
        options, arguments = getopt.getopt(sys.argv[1:], 'j:a:', ['--json', '--audio'])
    except getopt.GetoptError:
        print_usage()
    json_file = None
    audio_file = None

    for opt, arg in options:
        if opt in ('-j', '--json'):
            json_file = arg
        elif opt in ('-a', '--audio'):
            audio_file = arg

    if json_file is None or audio_file is None:
        print_usage()

    make_split(data_file=json_file, sound_file=audio_file)


"""Testing the writer, i.e., the deserialized-to-bibtex-converter."""

import pytest

from bibtexparser import BibtexFormat, Library, writer
from bibtexparser.model import (
    Entry,
    ExplicitComment,
    Field,
    ImplicitComment,
    ParsingFailedBlock,
    Preamble,
    String,
)

_DUMMY_STRING = String(key="myKey", value='"myValue"')
_DUMMY_PREAMBLE = Preamble(value='"myValue"')
_DUMMY_EXPLICIT_COMMENT = ExplicitComment(comment="myValue")
_DUMMY_IMPLICIT_COMMENT = ImplicitComment(comment="#myValue")

def _dummy_entry():
    return Entry(
        entry_type="article",
        key="myKey",
        fields=[
            Field(key="title", value='"myTitle"'),
            Field(key="author", value='"myAuthor"'),
        ],
    )

@pytest.mark.parametrize(
    "block, expected_string",
    [
        (_DUMMY_STRING, '@string{myKey = "myValue"}\n'),
        (_DUMMY_PREAMBLE, '@preamble{"myValue"}\n'),
        (_DUMMY_EXPLICIT_COMMENT, "@comment{myValue}\n"),
        (_DUMMY_IMPLICIT_COMMENT, "#myValue\n"),
    ],
)
def test_single_simple_blocks(block, expected_string):
    """Test the @string serializer."""
    library = Library(blocks=[block])
    string = writer.write(library)
    assert string == expected_string

@pytest.mark.parametrize("indent", [None, "", " ", "\t"])
def test_write_entry_with_indent(indent):
    entry_block = _dummy_entry()
    library = Library(blocks=[entry_block])
    bib_format = BibtexFormat()
    expected_indent = "\t"  # default
    if indent is not None:
        bib_format.indent = indent
        expected_indent = indent

    string = writer.write(library, bib_format)
    assert (
        string == f'@article{{myKey,\n{expected_indent}title = "myTitle",'
        f'\n{expected_indent}author = "myAuthor"\n}}\n'
    )

@pytest.mark.parametrize("trailing_comma", [None, True, False])
def test_write_entry_with_trailing_comma(trailing_comma):
    entry_block = _dummy_entry()
    library = Library(blocks=[entry_block])
    bib_format = BibtexFormat()
    expected = ""  # default
    if trailing_comma is not None:
        bib_format.trailing_comma = trailing_comma
        if trailing_comma:
            expected = ","

    string = writer.write(library, bib_format)
    assert (
        string == f'@article{{myKey,\n\ttitle = "myTitle",'
        f'\n\tauthor = "myAuthor"{expected}\n}}\n'
    )

@pytest.mark.parametrize("value_column", [None, 10, "auto"])
def test_entry_value_column(value_column):
    entry_block = _dummy_entry()
    entry_block.set_field(Field(key="veryverylongkeyfield", value="2020"))
    library = Library(blocks=[entry_block])
    bib_format = BibtexFormat()
    if value_column is not None:
        bib_format.value_column = value_column
    string = writer.write(library, bib_format)
    if value_column is None:
        # Make sure there are no unneeded spaces
        assert f'{bib_format.indent}title = "myTitle"' in string
        assert f'{bib_format.indent}author = "myAuthor"' in string
        assert f"{bib_format.indent}veryverylongkeyfield = 2020" in string
    elif value_column == 10:
        assert f'{bib_format.indent}title   = "myTitle"' in string
        assert f'{bib_format.indent}author  = "myAuthor"' in string
        assert f"{bib_format.indent}veryverylongkeyfield = 2020" in string
    if value_column == "auto":
        assert f'{bib_format.indent}title                = "myTitle"' in string
        assert f'{bib_format.indent}author               = "myAuthor"' in string
        assert f"{bib_format.indent}veryverylongkeyfield = 2020" in string

@pytest.mark.parametrize("block_separator", [None, "\n\n", "\n-----\n"])
def test_block_separator(block_separator):
    library = Library(blocks=[_DUMMY_STRING, _DUMMY_PREAMBLE])
    bib_format = BibtexFormat()
    if block_separator is not None:
        bib_format.block_separator = block_separator
    else:
        block_separator = "\n\n"  # default
    string = writer.write(library, bib_format)
    lines = string.splitlines()

    if block_separator == "\n":
        # Case where the next line immediately is the next block
        assert len(lines) == 2
        assert lines[0] == '@string{myKey = "myValue"}'
        assert lines[1] == '@preamble{"myValue"}'
    else:
        # Case where blocks are separated by at least one line
        expected_lines = block_separator.splitlines()
        for i, l in enumerate(expected_lines):
            assert lines[1 + i] == l

        assert lines[1 + len(expected_lines)] == '@preamble{"myValue"}'

def test_write_failed_block():
    raw = "@article{irrelevant-for-this-test,\nexcept = {that-there-need-to-be},\nother = {multiple-lines}\n}"
    block = ParsingFailedBlock(
        error=ValueError("Some error"), raw=raw, ignore_error_block=None
    )
    library = Library(blocks=[block])
    string = writer.write(library)
    lines = string.splitlines()

    assert len(lines) == 5
    assert lines[0].startswith("% WARNING")
    assert lines[1] == "@article{irrelevant-for-this-test,"
    assert lines[2] == "except = {that-there-need-to-be},"
    assert lines[3] == "other = {multiple-lines}"
    assert lines[4] == "}"


import numpy as np
import nibabel as ni
import os
import seaborn as sns

from mne.viz import circular_layout

import logging
logger = logging.getLogger(__name__)

currdir = os.path.dirname(os.path.abspath(__file__))
currdir = os.path.abspath(os.path.join(currdir, os.pardir))
atlasdir = os.path.join(currdir, 'io', 'data', 'atlas')

def find_roi_center(img, roi_value):
    """
    This function gives the x,y,z coordinates of a particular ROI using
    the given segmented image and the image level value used to select the ROI.

    Parameters
    ----------
    img : nibabel Nifti1Image instance
        The Nifti1Image instance of the segmented image.
    roi_value : int
        The value of the ROI as represented in the segmented image

    Returns
    -------
    xyz : tuple
        A triplets representing the xyz coordinate of the selected ROI.
    """

    affine = img.affine

    mask_ = np.int_(img.get_fdata()) == roi_value
    ijk_coords = np.array(np.nonzero(mask_)).mean(1)

    xyz_coords = ijk_coords * affine.diagonal()[:-1] + affine[:-1,-1]

    return xyz_coords

def get_aal_coords(fname):
    """Function used to obtain coordinates of the ROIs contained in the AAL90 atlas.
    The atlas used is the 2mm nifti version of the atlas.

    Parameters
    ----------
    fname : string
        The path of the atlas to be used.

    Returns
    -------
    coords : n x 3 numpy array
        The array containing n xyz coordinates in MNI space, one for each unique value of the atlas
    """

    atlas90 = ni.load(fname)
    coords = [find_roi_center(atlas90, roi_value=i) for i in np.unique(atlas90.get_data())[:]]

    return np.array(coords)

def get_findlab_coords():
    """
    Function used to obtain coordinates of the networks contained in the findlab atlas.
    The atlas used is the 2mm nifti version of the atlas.

    Returns
    -------
    coords : n x 3 numpy array
        The array containing n xyz coordinates in MNI space, one for each unique value of the atlas
    """
    atlas_dir = os.path.join(atlasdir, 'findlab')
    roi_list = os.listdir(atlas_dir)
    roi_list.sort()
    findlab = [ni.load(os.path.join(atlas_dir, roi)) for roi in roi_list if roi.find("nii") != -1]
    f_coords = []
    for img_ in findlab:

        centers = [find_roi_center(img_, roi_value=int(i)) for i in np.unique(img_.get_fdata())[1:]]
        f_coords.append(np.array(centers))

    return np.vstack(f_coords)

def get_atlas90_info(background='black'):

    """
    Utility function used to load informations about the atlas used

    Parameters
    ----------

    background : string | {'black', 'white'}
        A string used to build colors for plots.

    Returns
    -------
    names : list of string
        The list of ROI names.

    colors : list of string
        The list of colors used in other functions

    index_ : list of int
        How node values should be ordered if the atlas has another order
        (used to separate left/right in the atlas90)

    coords : list of tuple (x,y,z)
        Coordinates of the ROI center (used in plot_connectomics)

    networks : list of string
        The list of network names.

    """

    atlas_dir =  os.path.join(atlasdir, 'aal')
    coords = get_aal_coords(os.path.join(atlas_dir, "atlas90_mni_2mm.nii.gz"))
    roi_list = np.loadtxt(os.path.join(atlas_dir, "atlas90.cod"),
                            delimiter='=',
                            dtype=np.str)
    names = roi_list.T[1]
    names_inv = np.array([n[::-1] for n in names])
    index_ = np.argsort(names_inv)
    names_lr = names[index_]
    names = np.array([n.replace('_', ' ') for n in names])

    dict_ = {'L':'#89CC74',
             'R':'#7A84CC'}
    colors_lr = np.array([dict_[n[:1]] for n in names_inv])
    colors_lr = sns.husl_palette(2)
    networks = names

    node_angles = circular_layout(names.tolist(),
                                  names[index_].tolist(),
                                  start_pos=90,
                                  group_boundaries=[0, len(names) / 2.+1],
                                  group_sep=0.)

    return names, colors_lr, index_, coords, networks, node_angles

def get_findlab_info(background='black'):

    coords = get_findlab_coords()
    roi_file =  os.path.join(atlasdir, 'findlab', 'findlab_rois.txt')
    roi_list = np.loadtxt(roi_file, delimiter=',', dtype=str)
    networks = roi_list.T[-2]
    names = roi_list.T[2]

    dict_ = {
                'Auditory'          :'lightgray',
                'Basal_Ganglia'     :'lavender',
                #'Basal_Ganglia'    :'honeydew',
                'LECN'              :'tomato',
                'Language'          :'darkgrey',
                'Precuneus'         :'teal',
                'RECN'              :'lightsalmon',
                'Sensorimotor'      :'plum',
                'Visuospatial'      :'slateblue',
                'anterior_Salience' :'yellowgreen',
                #'dorsal_DMN'       :'lightsteelblue',
                'dorsal_DMN'        :'cadetblue',
                'high_Visual'       :'khaki',
                'post_Salience'     :'mediumseagreen',
                'prim_Visual'       :'gold',
                'ventral_DMN'       :'lightblue'
            }

    if background == 'white':
        dict_['anterior_Salience'] = 'gray'
        dict_['Basal_Ganglia'] = 'black'

    colors_lr = np.array([dict_[r.T[-2]] for r in roi_list])
    index_ = np.arange(90)

    _, count_ = np.unique(networks, return_counts=True)
    boundaries = np.cumsum(np.hstack(([0], count_)))[:-1]
    node_angles = circular_layout(names.tolist(),
                                  names.tolist(),
                                  start_pos=90,
                                  group_boundaries=boundaries.tolist(),
                                  group_sep=3.5)

    return names, colors_lr, index_, coords, networks, node_angles

def get_aalmeg_info(background='black', grouping='LR'):
    """[summary]

    Parameters
    ----------
    background : str, optional
        [description], by default 'black'
    grouping : str, optional
        [description], by default 'LR' or 'lobes'

    Returns
    -------
    [type]
        [description]
    """

    atlas_fname = os.path.join(atlasdir, 'aal', 'ROI_MNI_V4.nii')
    labels_fname = os.path.join(atlasdir, 'aal', 'ROI_MNI_V4.csv')
    coords = get_aal_coords(atlas_fname)

    labels = np.loadtxt(labels_fname, dtype=np.str, delimiter=',')
    node_names = labels.T[1][:99]

    if grouping == 'LR':
        node_idx = np.argsort(np.array([node[-1] for node in node_names]))
        group_boundaries = [0, len(node_names) / 2.+1]
        colors = sns.husl_palette(2)
        networks = node_names.copy()
    else:
        node_network = labels.T[3][:99]
        node_idx = np.argsort(node_network)
        networks, count = np.unique(node_network, return_counts=True)
        group_boundaries = np.cumsum(np.hstack(([0], count)))[:-1]
        color_network = sns.color_palette("Paired", len(networks)+1)
        colors_ = dict(zip(networks, color_network[1:]))
        colors = [colors_[n] for n in node_network]

    order = node_names[node_idx].tolist()
    node_angles = circular_layout(node_names.tolist(),
                                  order,
                                  start_pos=90,
                                  group_boundaries=group_boundaries,
                                  group_sep=3.
                                  )

    return labels, colors, node_idx, coords, networks, node_angles

def get_eeg_channels(n_channels=159):

    index_ = range(n_channels)
    names = list()
    colors_lr = list()
    coords = list()
    networks = list()
    for i in index_:
        names.append("ch-%03d" % (i+1))
        colors_lr.append("green")
        coords.append([0, 0, i])
        networks.append("Brain")

    node_angles = circular_layout(index_,
                                  index_,
                                  start_pos=90,
                                  group_sep=3.
                                  )

    return names, colors_lr, index_, coords, names, node_angles

def get_viviana_info():

    network = ['DAN','VAN','SMN','VIS','AUD','LAN','DMN']
    number = [6, 5, 8, 10, 4, 5, 7]
    color = ['lightgray', 'lavender', 'honeydew', 'tomato',
              'darkgrey', 'teal', 'lightsalmon']

    labels = ["%s_%02d" % (network[x], i+1) for x in range(len(network)) for i in range(number[x])]
    colors = [color[x] for x in range(len(network)) for i in range(number[x])]

    node_idx = np.arange(len(labels))

    coords = np.random.randint(-20, 20, (3, len(labels)))
    networks = [network[x] for x in range(len(network)) for i in range(number[x])]

    group_boundaries = np.cumsum(np.hstack(([0], number)))[:-1]
    node_angles = circular_layout(labels,
                                  labels,
                                  start_pos=90,
                                  group_boundaries=group_boundaries,
                                  group_sep=3.
                                  )

    return labels, colors, node_idx, coords, networks, node_angles

def get_atlas_info(atlas_name='findlab'):
    """This function is used to obtain information on
    different atlases.

    Parameters
    ----------
    atlas_name : str, optional ('findlab', 'aal_meg', 'atlas90')
        atlas name to get information

    Returns
    -------
    List of informations of the atlas:
        labels : names of nodes (n_nodes, string array)
        colors : colors of each node representing some classification
        node_idx : order of nodes for plot purposes
        coords : MNI coordinates of nodes
        networks : name of networks the node belongs to
        node_angles : angles for circular plot
    """

    mapper = {
        'findlab': get_findlab_info,
        'atlas90': get_atlas90_info,
        'aal_meg': get_aalmeg_info,
        'viviana': get_viviana_info,
        'eeg_channels': get_eeg_channels,

    }

    return mapper[atlas_name]()

def where_am_i(fname):
    """This function is used to find the ROI name in a mask file using
    AFNI whereami.

    Parameters
    ----------
    fname : string,
        The complete filename of the mask.

    Returns
    -------
    table : list
        The list has a lenght equal to the number of ROIs contained in the file
        Elements of the list:
        (x, y, z): coordinates of ROI's center of mass.
        label: name of the brain area closest to the center of mass
        num: number of voxels included in the ROI
        value: value of the ROI in the input file

    """

    logger.info("Looking for ROI names in "+fname)
    img = ni.load(fname)
    center_711 = np.array([-70.5, -105, -60.])
    data = img.get_data().squeeze()
    table = []
    for f in np.unique(data)[1:]:
        mask_roi = data == f

        center_mass = np.mean(np.nonzero(mask_roi), axis=1)
        x,y,z = np.rint([3,3,3]*center_mass+center_711)

        command = "whereami %s %s %s -lpi -space TLRC -tab" %(str(x+2.), str(y+2), str(z+2))
        var = os.popen(command).read()
        lines = var.split("\n")
        index = [i for i, l in enumerate(lines) if l[:5] == 'Atlas']
        label1 = lines[index[0]+1]
        label2 = lines[index[0]+2]
        if label1[0] == '*':
            area1 = area2 = "None"
        else:
            area1 = label1.split("\t")[2]
            area2 = label2.split("\t")[2]
        table.append([x, y, z, area1, np.count_nonzero(mask_roi), f])

    return table

def get_rois_names(path):
    import glob
    big_table = {}
    rois = glob.glob(path+"*mask.nii.gz")
    for fname in rois:
        table = where_am_i(fname)
        key = fname.split('/')[-1].split('.')[0][:-5]
        big_table[key] = table

    return big_table


"""
Featurizers that mostly concern protein-based models
"""
from __future__ import annotations
from collections import Counter
import logging
from typing import Union

import numpy as np

from .core import ParallelBaseFeaturizer, BaseOneHotEncodingFeaturizer, OEBaseModelingFeaturizer
from ..core.proteins import Protein, KLIFSKinase
from ..core.sequences import AminoAcidSequence
from ..core.systems import ProteinSystem, ProteinLigandComplex

logger = logging.getLogger(__name__)

class SingleProteinFeaturizer(ParallelBaseFeaturizer):
    """
    Provides a minimally useful ``._supports()`` method for all Protein-like featurizers.
    """

    _COMPATIBLE_PROTEIN_TYPES = (Protein, KLIFSKinase)

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _supports(self, system: Union[ProteinSystem, ProteinLigandComplex]) -> bool:
        """
        Check that exactly one protein is present in the System
        """
        super_checks = super()._supports(system)
        proteins = [c for c in system.components if isinstance(c, self._COMPATIBLE_PROTEIN_TYPES)]
        return all([super_checks, len(proteins) == 1])

class AminoAcidCompositionFeaturizer(SingleProteinFeaturizer):

    """Featurizes the protein using the composition of the residues in the binding site."""

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    # Initialize a Counter object with 0 counts
    _counter = Counter(sorted(AminoAcidSequence.ALPHABET))
    for k in _counter.keys():
        _counter[k] = 0

    def _featurize_one(
        self, system: Union[ProteinSystem, ProteinLigandComplex]
    ) -> Union[np.array, None]:
        """
        Featurizes a protein using the residue count in the sequence.

        Parameters
        ----------
        system: ProteinSystem or ProteinLigandComplex
            The System to be featurized.

        Returns
        -------
        : np.array or None
            The count of amino acids in the binding site.
        """
        count = self._counter.copy()
        try:
            sequence = system.protein.sequence
        except ValueError:  # e.g. erroneous uniprot_id in lazy instantiation
            return None
        count.update(system.protein.sequence)
        sorted_count = sorted(count.items(), key=lambda kv: kv[0])
        return np.array([number for _, number in sorted_count])

class OneHotEncodedSequenceFeaturizer(BaseOneHotEncodingFeaturizer, SingleProteinFeaturizer):

    """Featurizes the sequence of the protein to a one hot encoding."""

    ALPHABET = AminoAcidSequence.ALPHABET

    def __init__(self, sequence_type: str = "full", **kwargs):
        """
        Featurizes the sequence of the protein to a one hot encoding.

        Parameters
        ----------
        sequence_type: str, default=full
            The sequence to use for one hot encoding ('full', 'klifs_kinase' or 'klifs_structure').
        """
        if sequence_type not in ["full", "klifs_kinase", "klifs_structure"]:
            raise ValueError(
                "Only 'full', 'klifs_kinase' and 'klifs_structure' are supported sequence_types, "
                f"you provided {sequence_type}."
            )
        self.sequence_type = sequence_type
        if sequence_type != "full":
            self.ALPHABET += "-"  # add gap symbol for KLIFS sequence to ALPHABET
        super().__init__(**kwargs)  # update ALPHABET first

    def _retrieve_sequence(self, system: Union[ProteinSystem, ProteinLigandComplex]) -> str:
        try:
            if self.sequence_type == "full":
                sequence = system.protein.sequence
            elif self.sequence_type == "klifs_kinase":
                sequence = system.protein.kinase_klifs_sequence
            else:
                sequence = system.protein.structure_klifs_sequence
        except ValueError:  # e.g. erroneous uniprot_id in lazy instantiation
            return ""
        return sequence

class OEProteinStructureFeaturizer(OEBaseModelingFeaturizer, SingleProteinFeaturizer):
    """
    Given systems with exactly one protein, prepare the protein structure by:

     - modeling missing loops with OESpruce according to the PDB header unless
       a custom sequence is specified via the `uniprot_id` or `sequence`
       attribute in the protein component (see below), missing sequences at
       N- and C-termini are not modeled
     - building missing side chains
     - substitutions, deletions and insertions, if a `uniprot_id` or `sequence`
       attribute is provided for the protein component alteration will be
       modeled with OESpruce, if an alteration could not be modeled, the
       corresponding mismatch in the structure will be deleted
     - removing everything but protein and water
     - protonation at pH 7.4

    The protein component of each system must be a `core.proteins.Protein`
    or a subclass thereof, must be initialized with toolkit='OpenEye' and
    give access to a molecular structure, e.g. via a pdb_id. Additionally,
    the protein component can have the following optional attributes to
    customize the protein modeling:

     - `name`: A string specifying the name of the protein, will be used for
        generating the output file name.
     - `chain_id`: A string specifying which chain should be used.
     - `alternate_location`: A string specifying which alternate location
        should be used.
     - `expo_id`: A string specifying a ligand bound to the protein of
       interest. This is especially useful if multiple proteins are found in
       one PDB structure.
     - `uniprot_id`: A string specifying the UniProt ID that will be used to
       fetch the amino acid sequence from UniProt, which will be used for
       modeling the protein. This will supersede the sequence information
       given in the PDB header.
     - `sequence`: A  string specifying the amino acid sequence in
       one-letter-codes that should be used during modeling the protein. This
       will supersede a given `uniprot_id` and the sequence information given
       in the PDB header.

    Parameters
    ----------
    loop_db: str
        The path to the loop database used by OESpruce to model missing loops.
    cache_dir: str, Path or None, default=None
        Path to directory used for saving intermediate files. If None, default
        location provided by `appdirs.user_cache_dir()` will be used.
    output_dir: str, Path or None, default=None
        Path to directory used for saving output files. If None, output
        structures will not be saved.
    use_multiprocessing : bool, default=True
        If multiprocessing to use.
    n_processes : int or None, default=None
        How many processes to use in case of multiprocessing. Defaults to
        number of available CPUs.
    """

    from MDAnalysis.core.universe import Universe

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _featurize_one(self, system: ProteinSystem) -> Union[Universe, None]:
        """
        Prepare a protein structure.

        Parameters
        ----------
        system: ProteinSystem
            A system object holding a protein component.

        Returns
        -------
        : Universe or None
            An MDAnalysis universe of the featurized system. None if no design unit was found.
        """
        from pathlib import Path

        from ..modeling.MDAnalysisModeling import read_molecule

        structure = self._read_protein_structure(system.protein)
        if structure is None:
            logger.warning(
                f"Could not read protein structure for {system.protein}, returning None!"
            )
            return None

        logging.debug("Preparing protein structure ...")
        design_unit = self._get_design_unit(
            structure=structure,
            chain_id=system.protein.chain_id if hasattr(system.protein, "chain_id") else None,
            alternate_location=system.protein.alternate_location
            if hasattr(system.protein, "alternate_location")
            else None,
            has_ligand=hasattr(system.protein, "expo_id"),
            ligand_name=system.protein.expo_id if hasattr(system.protein, "expo_id") else None,
            model_loops_and_caps=False if system.protein.sequence else True,
        )  # if sequence is given model loops and caps separately later
        if not design_unit:
            logging.debug("No design unit found, returning None!")
            return None

        logging.debug("Extracting design unit components ...")
        protein, solvent = self._get_components(
            design_unit=design_unit,
            chain_id=system.protein.chain_id if hasattr(system.protein, "chain_id") else None,
        )[:-1]

        if system.protein.sequence:
            first_id = 1
            if "construct_range" in system.protein.metadata.keys():
                first_id = int(system.protein.metadata["construct_range"].split("-")[0])
            protein = self._process_protein(
                protein_structure=protein,
                amino_acid_sequence=system.protein.sequence,
                first_id=first_id,
            )

        logging.debug("Assembling components ...")
        solvated_protein = self._assemble_components(protein, solvent)

        logging.debug("Updating pdb header ...")
        solvated_protein = self._update_pdb_header(
            solvated_protein, protein_name=system.protein.name
        )

        logging.debug("Writing results ...")
        file_path = self._write_results(
            solvated_protein,
            "_".join(
                [
                    info
                    for info in [
                        system.protein.name,
                        system.protein.pdb_id
                        if system.protein.pdb_id
                        else Path(system.protein.metadata["file_path"]).stem,
                        f"chain{system.protein.chain_id}"
                        if hasattr(system.protein, "chain_id")
                        else None,
                        f"altloc{system.protein.alternate_location}"
                        if hasattr(system.protein, "alternate_location")
                        else None,
                    ]
                    if info
                ]
            ),
        )

        logging.debug("Generating new MDAnalysis universe ...")
        structure = read_molecule(file_path)

        if not self.output_dir:
            logging.debug("Removing structure file ...")
            file_path.unlink()

        return structure


import streamlit as st
import pandas as pd
import plotly.express as px

df = pd.read_excel('D:\dakeshna\Django Python\phonepe.xlsx')

st.set_page_config(page_title='My Dashboard')
st.title('My Dashboard')

state = st.sidebar.selectbox('Select a state', df['state'].unique())

filtered_df = df[df['state'] == state]

fig = px.bar(filtered_df, x='transaction_type', y='transaction_count', color='year',
             title='Transaction Count by Transaction Type')
st.plotly_chart(fig)

fig = px.line(filtered_df, x='year', y='transaction_amount', title='Transaction Amount Over Time')
st.plotly_chart(fig)

fig = px.scatter(filtered_df, x='transaction_count', y='transaction_amount', color='year',
                 title='Transaction Count vs. Transaction Amount')
st.plotly_chart(fig)


# -*- coding: utf-8 -*-

from collections import defaultdict
from collections import Counter
import time
import math
import sys
from nodo import NodoDT

"""
Comprueba si un nodo de un arbol es interior
"""

def es_interior(nodo):
    if nodo.ramas:
        return len([rama for rama in nodo.ramas if rama]) > 0
    else:
        return False

'''
Obtiene el nodo raiz de un nuevo arbol que llega hasta el nodo intermedio indicado, que se convertirá en un nodo hoja.
'''

def obtener_nuevo_arbol(raiz, objetivo):
    return obtener_nuevo_arbol_recursivo(raiz, objetivo)

def obtener_nuevo_arbol_recursivo(actual, objetivo):
    if actual == objetivo:
        key = maxima_frecuencia(objetivo.distr)
        nodo = NodoDT(None, objetivo.distr, None, key)
    else:
        if actual.ramas:
            for key in actual.ramas:
                hijo = actual.ramas[key]
                actual.ramas[key] = obtener_nuevo_arbol_recursivo(hijo, objetivo)

        nodo = actual

    return nodo

"""
Crea un diccionario donde la clave es el nombre de la clase
y el valor es el número de veces que aparece esta.
"""

def distribucion_clases(datos):
    res = defaultdict(int)

    [res.update({x[-1]: res[x[-1]] + 1}) for x in datos]

    return res

"""
Crea un diccionario donde la clave es el nombre del atributo y el valor
es el índice de este.
"""

def indice_atributos(atributos):
    res = dict()

    [res.update({x[0]: atributos.index(x)}) for x in atributos]

    return res

"""
Función utilizada para capturar el momento en el que comienza todo.
"""

def comienzo_tiempo_ejecucion():
    # Variable usada para medir los tiempos de ejecución
    start_time = time.time()
    return start_time

"""
Función utilizada para calcular los tiempos de ejecución de la aplicación en base al comienzo.
"""

def tiempo_ejecucion_obtenido(start_time):
    # Tiempo de ejecución obtenido
    print("Tiempo de ejecución en segundos: --- %s seconds ---" % (time.time() - start_time))

"""
Calcula el total de datos pasando una distribución.
"""

def total_datos_distribucion(distribucion):
    return sum([distribucion[key] for key in distribucion])

"""
Calcula la proporción de los datos dados.
"""

def proporcion_datos(datos):
    result = dict()

    total = sum([datos[x] for x in datos])

    for key in datos:
        result[key] = datos[key] / total

    return result

"""
Devuelve la clave de la máxima frecuencia, dado un conjunto de datos de ejemplo
"""

def maxima_frecuencia(datos):
    return max(datos, key=datos.get)

"""
Calcula la tasa de error de un conjunto de datos.
"""

def tasa_error(S):
    total = total_datos_distribucion(S)
    key = max(S, key=S.get)
    Pd = S.get(key)

    return 1 - (Pd / total)

def tasa_error_media_ponderada(S, Si):
    total_S = total_datos_distribucion(S)
    total_Si = total_datos_distribucion(Si)

    return (total_Si / total_S) * tasa_error(Si)

"""
Calcula el índice de gini de un conjunto de datos.
"""

def gini(S):
    res = 0
    total = total_datos_distribucion(S)

    for key in S:
        p = S[key] / total
        res += math.pow(p, 2)

    return 1 - res

def gini_media_ponderada(S, Si):
    total_S = total_datos_distribucion(S)
    total_Si = total_datos_distribucion(Si)

    return (total_Si / total_S) * gini(Si)

"""
Calcula la entropia de un conjunto de datos dada la distribución total.
"""

def entropia(S):
    res = 0
    total = total_datos_distribucion(S)

    for key in S:
        p = S[key] / total
        res += p * math.log(p, 2)

    return -res

"""
Calcula el grado de entropía del criterio de decisión.
Media ponderada de los grados de entropía de los conjuntos obtenidos tras la decisión.
"""

def entropia_media_ponderada(S, Si):
    total_S = total_datos_distribucion(S)
    total_Si = total_datos_distribucion(Si)

    return (total_Si / total_S) * entropia(Si)

"""
Calcula la media ponderada según el criterio de medida
"""

def calcular_media_ponderada(medida, S, Si):
    if medida == "entropia":
        resultado = entropia_media_ponderada(S, Si)
    elif medida == "error":
        resultado = tasa_error_media_ponderada(S, Si)
    elif medida == "gini":
        resultado = gini_media_ponderada(S, Si)
    else:
        raise ValueError("La medida {} no está registrada.".format(medida))

    return resultado

"""
Selecciona la mejor opción para elegir qué rama es mejor explorar
"""

def criterio_decision(medida, datos, atributos, indice_atributos):
    S = distribucion_clases(datos)
    Si = []
    Si_dict = dict()

    # Buscamos en cada atributo restante
    for atributo, indice in indice_atributos.items():
        for valor in atributos[indice][1]:
            # Cogemos los datos correspondientes a ese atributo y valor
            datos_filtrados = [dato for dato in datos if dato[indice] == valor]

            # Realizamos la distribución de cada conjunto de datos
            Si.append(distribucion_clases(datos_filtrados))

        Si_dict.update({atributo: Si})
        Si = []

    """
    Declaramos la variable total para calcular cada una de las medias
    ponderadas
    """
    total = 0
    minimo = sys.maxsize
    mejor_atributo = ""

    for atributo in Si_dict:
        for Si in Si_dict[atributo]:

            # Si 'Si' no está vacio calculo su media
            if not not Si:
                total += calcular_media_ponderada(medida, S, Si)

        if total <= minimo:
            minimo = total
            mejor_atributo = atributo

    return mejor_atributo

"""
Obtiene un nuevo conjunto de datos
"""

def filtrar_nuevos_datos(datos, valor, indice):
    nuevos_datos = []

    for dato in datos:
        if valor in dato[indice]:
            nuevos_datos.append(dato)

    return nuevos_datos

"""
Obtener la categoría con más frecuencia
"""

def obtener_frecuencia_ordenada(lista):
    dict_aux = distribucion_clases(lista)
    resultado = dict()
    [resultado.update({k: dict_aux[k]}) for k in sorted(dict_aux, key=dict_aux.get)]
    return resultado

"""
Devuelve los ejemplos cubiertos que quedan cubiertos por la regla R en el conjunto de datos D
"""

def ejemplos_cubiertos(R, D):
    ejemplos_cubiertos = D

    for condicion in R:
        indice_atributo, valor = condicion
        ejemplos_cubiertos = [ejemplo for ejemplo in ejemplos_cubiertos if ejemplo[indice_atributo] == valor]

    return ejemplos_cubiertos

"""
Devuelve los ejemplos que están correctamente cubiertos por la regla R en el conjunto de datos D, y que coincidan con
la clase C.
"""

def ejemplos_correctamente_cubiertos(R, D, C):
    ejemplos = ejemplos_cubiertos(R, D)
    return filtrar_clase(ejemplos, C)

"""
Devuelve la frecuencia relativa para la regla R, en el conjunto de datos D, y para la clase C
"""

def frecuencia_relativa(R, D, C):
    t = len(ejemplos_cubiertos(R, D))
    p = len(ejemplos_correctamente_cubiertos(R, D, C))

    return p / t

"""
Una funcion que dada los atributos A y un conjuntos de datos D, devuelva una regla para ese conjunto de datos
"""

# =============================================================================
#         Una regla tiene la siguiente forma:
#         reglas = [
#             ([(1, 'uno'), (0, 'parado'), (3, 'dos o mas')], 'estudiar'),
#             ([(1, 'dos'), (0, 'parado'), (3, 'dos o mas')], 'no conceder'),
#         ]
# =============================================================================
def obtener_regla(A, D):
    indices = indice_atributos(A)
    regla_list = list()
    max_ejemplos_cubiertos = 0
    tupla_max = tuple()

    for atributo in indices:

        for valor_atributo in A[indices[atributo]][1]:

            # Genero la tupla con el indice del atributo y su valor
            tupla = (indices[atributo], valor_atributo)

            # Añado la tupla nueva a la copia de la lista
            regla_list.append(tupla)

            # Calculo cuantos ejemplos cubrimos con dicha regla
            ej_cubiertos = len(ejemplos_cubiertos(regla_list, D))

            # Compruebo si el recubrimiento de ejemplos es más grande que el maximo
            if ej_cubiertos > max_ejemplos_cubiertos:
                max_ejemplos_cubiertos = ej_cubiertos
                tupla_max = tupla

            # Elimino la tupla que insertamos al principio
            regla_list.remove(regla_list[-1])

        regla_list.append(tupla_max)
        max_ejemplos_cubiertos = 0

    return regla_list

"""
Función encargada de filtrar un conjunto de entrenamiento dada una clase
"""

def filtrar_clase(lista, clase):
    return [item for item in lista if item[-1] == clase]

"""
Una funcion que para una clase entera, devuelva todas las reglas de esa clase,
itera sobre la anterior y con el resto de atributos que no estén cubiertos por esa primera regla, se llama a la anterior
con el conjunto de ejemplos restante para crear otra regla, hasta que no queden ejemplos
"""

def obtener_reglas_por_clases(A, D, C):
    filtro_conjunto_datos = filtrar_clase(D, C)
    reglas_list = list()

    while len(filtro_conjunto_datos) != 0:
        regla = (obtener_regla(A, filtro_conjunto_datos), C)
        reglas_list.append(regla)
        filtro_conjunto_datos = list(
            filter(lambda x: x not in ejemplos_cubiertos(regla[0], filtro_conjunto_datos), filtro_conjunto_datos))

    return reglas_list

"""
Por último, otra que itere sobre todas las clases y llame a la anterior, pero en esta funcion en concreto no se reducen
los ejemplos.
"""

def obtener_total_reglas(A, D):
    reglas_list = list()

    C =list(obtener_frecuencia_ordenada(D).keys())

    # Cogemos todas las clases menos la última que será la de por defecto
    for clase in C[:-1]:
        reglas_list.append(obtener_reglas_por_clases(A, D, clase))

    # Aplanar lista de listas
    reglas_list = sum(reglas_list, [])

    # Añadimos la última regla por defecto
    reglas_list.append((None, C[-1]))

    return reglas_list


# -*- coding:UTF-8  -*-
"""
http://www.94xmn.com/
@author: hikaru
email: hikaru870806@hotmail.com
如有问题或建议请联系
"""
import os
import re
import time
import traceback
from pyquery import PyQuery as pq
from common import *

SUB_PATH_LIST = {
    "xiaoqingxin": "1",
    "xingganmeinv": "2",
    "siwameitui": "3",
    "ribenmeinv": "4",
    "hanguomeinv": "5",
    "xioumeinv": "6",
    "jiepai": "8",
}

# 获取指定一页的图集
def get_one_page_album(sub_path, page_count):
    album_pagination_url = "http://www.94xmn.com/%s/list_%s_%s.html" % (sub_path, SUB_PATH_LIST[sub_path], page_count)
    album_pagination_response = net.http_request(album_pagination_url, method="GET", header_list={"Host": "www.94xmn.com"})
    result = {
        "album_info_list": {},  # 全部图集信息
        "is_over": False,  # 是否最后一页图集
    }
    if album_pagination_response.status == 409:
        log.step("分页图集 %s 访问异常，重试" % album_pagination_url)
        time.sleep(5)
        return get_one_page_album(sub_path, page_count)
    elif album_pagination_response.status != net.HTTP_RETURN_CODE_SUCCEED:
        raise crawler.CrawlerException(crawler.request_failre(album_pagination_response.status))
    # 页面编码
    album_pagination_response_content = album_pagination_response.data.decode(errors="ignore")
    # 获取图集信息，存在两种页面样式
    album_list_selector = pq(album_pagination_response_content).find(".wf-main .wf-cld a")
    if album_list_selector.length == 0:
        raise crawler.CrawlerException("页面截取图集列表失败\n%s" % album_pagination_response_content)
    for album_index in range(0, album_list_selector.length):
        result_album_info = {
            "album_id": None,  # 图集id
            "album_title": "",  # 图集id
            "album_url": None,  # 图集地址
        }
        album_selector = album_list_selector.eq(album_index)
        # 获取图集地址
        album_url = album_selector.attr("href")
        if not album_url:
            raise crawler.CrawlerException("图集列表截取图集地址失败\n%s" % album_selector.html())
        result_album_info["album_url"] = album_url
        # 获取图集id
        # http://www.94xmn.com/plus/view.php?aid=25
        if album_url.find("/view.php?") > 0:
            album_id = album_url.split("aid=")[-1]
        else:
            album_id = album_url.split("/")[-1].split(".")[0]
        if not crawler.is_integer(album_id):
            raise crawler.CrawlerException("图集地址截取图集id失败\n%s" % album_url)
        result_album_info["album_id"] = int(album_id)
        # 获取图集标题
        album_title = album_selector.attr("title")
        if not album_title:
            raise crawler.CrawlerException("图集列表截取图集标题失败\n%s" % album_url)
        result_album_info["album_title"] = album_title
        result["album_info_list"][result_album_info["album_id"]] = result_album_info
    # 判断是不是最后一页
    max_page_count = pq(album_pagination_response_content).find("div.page li:last span.pageinfo strong:first").html()
    if not crawler.is_integer(max_page_count):
        raise crawler.CrawlerException("页面截取总页数失败\n%s" % album_pagination_response_content)
    result["is_over"] = page_count >= int(max_page_count)
    return result

# 获取指定图集
def get_album_page(album_url):
    page_count = max_page_count = 1
    result = {
        "photo_url_list": [],  # 全部图片地址
    }
    while page_count <= max_page_count:
        if page_count == 1:
            pagination_album_url = album_url
        else:
            pagination_album_url = album_url.replace(".html", "_%s.html" % page_count)
        album_response = net.http_request(pagination_album_url, method="GET", header_list={"Host": "www.94xmn.com"})
        if album_response.status == 409:
            log.step("图集分页 %s 访问异常，重试" % pagination_album_url)
            time.sleep(5)
            continue
        elif album_response.status != net.HTTP_RETURN_CODE_SUCCEED:
            raise crawler.CrawlerException("第%s页" % page_count + crawler.request_failre(album_response.status))
        album_response_content = album_response.data.decode(errors="ignore")
        # 获取图片地址
        photo_list_selector = pq(album_response_content).find("div.bbox a img")
        if photo_list_selector.length == 0:
            raise crawler.CrawlerException("第%s页页面匹配图片地址失败\n%s" % (page_count, album_response_content))
        for photo_index in range(0, photo_list_selector.length):
            result["photo_url_list"].append("http://www.94xmn.com" + photo_list_selector.eq(photo_index).attr("src"))
        # 获取总页数
        if page_count == 1:
            max_page_html = pq(album_response_content).find("#fenye li:first a").html()
            if not max_page_html:
                raise crawler.CrawlerException("页面匹配总页数信息失败\n%s" % album_response_content)
            max_page_count_find = re.findall("共(\d*)页: ", max_page_html)
            if len(max_page_count_find) == 0:
                raise crawler.CrawlerException("总页数信息匹配总页数失败\n%s" % album_response_content)
            max_page_count = int(max_page_count_find[0])
        page_count += 1
    return result

class Album(crawler.Crawler):
    def __init__(self, **kwargs):
        # 设置APP目录
        crawler.PROJECT_APP_PATH = os.path.abspath(os.path.dirname(__file__))

        # 初始化参数
        sys_config = {
            crawler.SYS_DOWNLOAD_PHOTO: True,
            crawler.SYS_NOT_CHECK_SAVE_DATA: True,
        }
        crawler.Crawler.__init__(self, sys_config, **kwargs)

        # 解析存档文件
        # sub_path  last_album_id
        self.account_list = crawler.read_save_data(self.save_data_path, 0, ["", "0"])
        for sub_path in SUB_PATH_LIST:
            if sub_path not in self.account_list:
                self.account_list[sub_path] = [sub_path, "0"]

    def main(self):
        # 循环下载每个id
        thread_list = []
        for sub_path in sorted(self.account_list.keys()):
            # 提前结束
            if not self.is_running():
                break

            # 开始下载
            thread = Download(self.account_list[sub_path], self)
            thread.start()
            thread_list.append(thread)

            time.sleep(1)

        # 等待子线程全部完成
        while len(thread_list) > 0:
            thread_list.pop().join()

        # 未完成的数据保存
        if len(self.account_list) > 0:
            file.write_file(tool.list_to_string(list(self.account_list.values())), self.temp_save_data_path)

        # 重新排序保存存档文件
        crawler.rewrite_save_file(self.temp_save_data_path, self.save_data_path)

        log.step("全部下载完毕，耗时%s秒，共计图片%s张" % (self.get_run_time(), self.total_photo_count))

class Download(crawler.DownloadThread):
    def __init__(self, account_info, main_thread):
        crawler.DownloadThread.__init__(self, account_info, main_thread)
        self.sub_path = self.account_info[0]
        self.display_name = self.sub_path
        self.step("开始")

    # 获取所有可下载图集
    def get_crawl_list(self):
        page_count = 1
        album_info_list = []
        temp_album_info_list = {}
        is_over = False
        # 获取全部图集
        while not is_over:
            self.main_thread_check()  # 检测主线程运行状态
            self.step("开始解析第%s页图集" % page_count)

            # 获取一页图集
            try:
                album_response = get_one_page_album(self.sub_path, page_count)
            except crawler.CrawlerException as e:
                self.error("第%s页图集解析失败，原因：%s" % (page_count, e.message))
                raise

            self.trace("第%s页解析的全部图集：%s" % (page_count, album_response["album_info_list"]))
            self.step("第%s页解析获取%s个图集" % (page_count, len(album_response["album_info_list"])))

            temp_album_info_list.update(album_response["album_info_list"])

            if album_response["is_over"]:
                is_over = True
            else:
                page_count += 1

        # 获取全部还未下载过需要解析的图集
        for album_id in sorted(temp_album_info_list.keys(), reverse=True):
            # 检查是否达到存档记录
            if album_id > int(self.account_info[1]):
                album_info_list.append(temp_album_info_list[album_id])
            else:
                break

        return album_info_list

    # 解析单个图集
    def crawl_album(self, album_info):
        self.step("开始解析图集%s %s" % (album_info["album_id"], album_info["album_url"]))

        # 获取图集全部图片
        try:
            album_response = get_album_page(album_info["album_url"])
        except crawler.CrawlerException as e:
            self.error("图集%s解析失败，原因：%s" % (album_info["album_id"], e.message))
            raise

        self.trace("图集%s《%s》 %s 解析的全部图片：%s" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], album_response["photo_url_list"]))
        self.step("图集%s《%s》 %s 解析获取%s张图片" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], len(album_response["photo_url_list"])))

        photo_index = 1
        # 过滤标题中不支持的字符
        album_title = path.filter_text(album_info["album_title"])
        if album_title:
            album_path = os.path.join(self.main_thread.photo_download_path, "%04d %s" % (album_info["album_id"], album_title))
        else:
            album_path = os.path.join(self.main_thread.photo_download_path, "%04d" % album_info["album_id"])
        # 设置临时目录
        self.temp_path_list.append(album_path)
        for photo_url in album_response["photo_url_list"]:
            self.main_thread_check()  # 检测主线程运行状态
            self.step("图集%s《%s》开始下载第%s张图片 %s" % (album_info["album_id"], album_info["album_title"], photo_index, photo_url))

            file_path = os.path.join(album_path, "%03d.%s" % (photo_index, net.get_file_type(photo_url)))
            save_file_return = net.save_net_file(photo_url, file_path)
            if save_file_return["status"] == 1:
                self.step("图集%s《%s》第%s张图片下载成功" % (album_info["album_id"], album_info["album_title"], photo_index))
                photo_index += 1
            else:
                self.error("图集%s《%s》 %s 第%s张图片 %s 下载失败，原因：%s" % (album_info["album_id"], album_info["album_title"], album_info["album_url"], photo_index, photo_url, crawler.download_failre(save_file_return["code"])))

        # 图集内图片全部下载完毕
        self.temp_path_list = []  # 临时目录设置清除
        self.total_photo_count += photo_index - 1  # 计数累加
        self.account_info[1] = str(album_info["album_id"])  # 设置存档记录

    def run(self):
        try:
            # 获取所有可下载图集
            album_info_list = self.get_crawl_list()
            self.step("需要下载的全部图集解析完毕，共%s个" % len(album_info_list))

            # 从最早的图集开始下载
            while len(album_info_list) > 0:
                self.crawl_album(album_info_list.pop())
                self.main_thread_check()  # 检测主线程运行状态
        except SystemExit as se:
            if se.code == 0:
                self.step("提前退出")
            else:
                self.error("异常退出")
            # 如果临时目录变量不为空，表示某个图集正在下载中，需要把下载了部分的内容给清理掉
            self.clean_temp_path()
        except Exception as e:
            self.error("未知异常")
            self.error(str(e) + "\n" + traceback.format_exc(), False)

        # 保存最后的信息
        with self.thread_lock:
            file.write_file("\t".join(self.account_info), self.main_thread.temp_save_data_path)
            self.main_thread.total_photo_count += self.total_photo_count
            self.main_thread.account_list.pop(self.sub_path)
        self.step("下载完毕，总共获得%s张图片" % self.total_photo_count)
        self.notify_main_thread()

if __name__ == "__main__":
    Album().main()


import torch
from classifier import ColorClassifier
from dataset_det import Balls_CF_Detection
from torch.utils.tensorboard import SummaryWriter
import numpy as np

# environment settings
device = torch.device("cuda" if torch.cuda.is_available() else 'cpu')
print(torch.cuda.get_device_name(torch.cuda.current_device()))
writer = SummaryWriter("runs/color_classifier.01")

# dataset
train_dataset = Balls_CF_Detection("train", 18899, train=1)
valid_dataset = Balls_CF_Detection("valid", 2099)

# dataloader
training = torch.utils.data.DataLoader(train_dataset,
                                       batch_size=50, shuffle=True)
validation = torch.utils.data.DataLoader(valid_dataset,
                                         batch_size=50, shuffle=True)

# model
model = ColorClassifier().to(device)
model.cuda()
criterion = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

batch_size_1 = 378
batch_size_2 = 42
dataset_1 = 18900 * 9
dataset_2 = 2100 * 9
train_loss = 0
train_error = 0
validation_loss = 0
validation_error = 0

for epoch in range(200):
    print("starting epoch:", epoch, "loss:", "training", train_loss, "validation", validation_loss, "error:",
          "training", train_error, "validation", validation_error)

    train_loss = 0
    train_error = 0
    validation_loss = 0
    validation_error = 0

    for batch, (train_img, train_presence, train_box) in enumerate(training):

        if (batch % 50) == 0:
            print("batch:", batch)

        optimizer.zero_grad()
        out1 = model(train_img.to(device))

        loss = criterion(out1.to(device), train_presence.to(device))
        train_loss += loss.item()

        loss.backward()
        optimizer.step()

        out1[out1 > 0.5] = 1
        out1[out1 < 0.5] = 0
        train_error += (out1 != train_presence.to(device)).sum().item()

    for batch, (valid_img, valid_presence, valid_box) in enumerate(validation):

        if (batch % 20) == 0:
            print("batch:", batch)

        out2 = model(valid_img.to(device))

        loss = criterion(out2.to(device), valid_presence.to(device))
        validation_loss += loss.item()

        out2[out2 > 0.5] = 1
        out2[out2 < 0.5] = 0
        validation_error += (out2 != valid_presence.to(device)).sum().item()

    train_loss = train_loss / batch_size_1
    train_error = train_error / dataset_1

    validation_loss = validation_loss / batch_size_2
    validation_error = validation_error / dataset_2

    writer.add_scalars("loss", {"training": train_loss, "validation": validation_loss}, epoch)
    writer.add_scalars("error", {"training": train_error, "validation": validation_error}, epoch)

# save the results
torch.save({"state_dict": model.state_dict(),
            "epoch": 200},
           "colorClassifier.pth")
writer.close()


#importing necessary libraries
import pandas as pd
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier

# visualization
import seaborn as sns
import matplotlib.pyplot as plt

#dataset
train_df = pd.read_csv('train.csv')
test_df = pd.read_csv('test.csv')
combine = [train_df, test_df]

print(train_df.columns.values)
train_df.head()
train_df.tail()

train_df.info()
train_df.describe()

train_df[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["Sex", "Survived"]].groupby(['Sex'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["SibSp", "Survived"]].groupby(['SibSp'], as_index=False).mean().sort_values(by='Survived', ascending=False)

train_df[["Parch", "Survived"]].groupby(['Parch'], as_index=False).mean().sort_values(by='Survived', ascending=False)
#SibSp and Parch These features have zero correlation for certain values. It is best to derive a feature or a set of features from these individual features

#Correlating numerical features
g = sns.FacetGrid(train_df, col='Survived')
g.map(plt.hist, 'Age', bins=20)

#Correlating numerical and ordinal features
grid = sns.FacetGrid(train_df, col='Survived', row='Pclass', size=2.2, aspect=1.6)
grid.map(plt.hist, 'Age', alpha=.5, bins=20)
grid.add_legend();

#Correlating categorical features
grid = sns.FacetGrid(train_df, row='Embarked', size=2.2, aspect=1.6)
grid.map(sns.pointplot, 'Pclass', 'Survived', 'Sex', palette='deep')
grid.add_legend()

#Correlating categorical and numerical features
grid = sns.FacetGrid(train_df, row='Embarked', col='Survived', size=2.2, aspect=1.6)
grid.map(sns.barplot, 'Sex', 'Fare', alpha=.5, ci=None)
grid.add_legend()

#dropping features
print("Before", train_df.shape, test_df.shape, combine[0].shape, combine[1].shape)
train_df = train_df.drop(['Ticket', 'Cabin'], axis=1)
test_df = test_df.drop(['Ticket', 'Cabin'], axis=1)
combine = [train_df, test_df]
print("After", train_df.shape, test_df.shape, combine[0].shape, combine[1].shape)

for dataset in combine:
    dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\.', expand=False)

pd.crosstab(train_df['Title'], train_df['Sex'])

#We can replace many titles with a more common name or classify them as Rare.
for dataset in combine:
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',
 	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer'], 'Rare')

    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')  #Mlle french for miss
    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')    #Mme equivalent for mrs

train_df[['Title', 'Survived']].groupby(['Title'], as_index=False).mean()

#converting the categorical titles to ordinal.
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
    dataset['Title'] = dataset['Title'].map(title_mapping)
    dataset['Title'] = dataset['Title'].fillna(0)

train_df.head()

#Now we can safely drop the Name feature from training and testing datasets. We also do not need the PassengerId feature in the training dataset.
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
test_df = test_df.drop(['Name'], axis=1)
combine = [train_df, test_df]
train_df.shape, test_df.shape

#Converting a categorical feature
for dataset in combine:
    dataset['Sex'] = dataset['Sex'].map( {'female': 1, 'male': 0} ).astype(int)

train_df.head()

#Completing a numerical continuous feature
grid = sns.FacetGrid(train_df, row='Pclass', col='Sex', size=2.2, aspect=1.6)
grid.map(plt.hist, 'Age', alpha=.5, bins=20)
grid.add_legend()

#Let we start by preparing an empty array to contain guessed Age values based on Pclass x Gender combinations.
guess_ages = np.zeros((2,3))
guess_ages

#Now we iterate over Sex (0 or 1) and Pclass (1, 2, 3) to calculate guessed values of Age for the six combinations.
for dataset in combine:
    for i in range(0, 2):
        for j in range(0, 3):
            guess_df = dataset[(dataset['Sex'] == i) &
                                  (dataset['Pclass'] == j+1)]['Age'].dropna()

            age_guess = guess_df.median()

            # Convert random age float to nearest .5 age
            guess_ages[i,j] = int( age_guess/0.5 + 0.5 ) * 0.5

    for i in range(0, 2):
        for j in range(0, 3):
            dataset.loc[ (dataset.Age.isnull()) & (dataset.Sex == i) & (dataset.Pclass == j+1),'Age'] = guess_ages[i,j]
    dataset['Age'] = dataset['Age'].astype(int)

train_df.head()

#creating Age bands and determine correlations with Survived.
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False).mean().sort_values(by='AgeBand', ascending=True)

#Replacing Age with ordinals based on these bands.
for dataset in combine:
    dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1
    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2
    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3
    dataset.loc[ dataset['Age'] > 64, 'Age'] =4
train_df.head()

#We can now remove the AgeBand feature.
train_df = train_df.drop(['AgeBand'], axis=1)
combine = [train_df, test_df]
train_df.head()

#Creating new feature combining existing features
for dataset in combine:
    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1

train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False).mean().sort_values(by='Survived', ascending=False)

#We can create another feature called IsAlone.
for dataset in combine:
    dataset['IsAlone'] = 0
    dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1

train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False).mean()

#droping Parch, SibSp, and FamilySize features in favor of IsAlone.
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
combine = [train_df, test_df]
train_df.head()

#We can also create a feature combining Pclass and Age.
for dataset in combine:
    dataset['Age*Class'] = dataset.Age * dataset.Pclass

train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10)

#Completing a categorical feature
#Embarked feature takes S, Q, C values based on port of embarkation.
freq_port = train_df.Embarked.dropna().mode()[0]
freq_port
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)

train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False).mean().sort_values(by='Survived', ascending=False)

#Converting categorical feature to numeric
for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].map( {'S': 0, 'C': 1, 'Q': 2} ).astype(int)
train_df.head()

#We can now complete the Fare feature for single missing value in test dataset using median to get the value for this feature
test_df['Fare'].fillna(test_df['Fare'].dropna().median(), inplace=True)
test_df.head()

#We can now create FareBand.
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False).mean().sort_values(by='FareBand', ascending=True)

#Converting the Fare feature to ordinal values based on the FareBand.
for dataset in combine:
    dataset.loc[(dataset['Fare'] > -0.001) &(dataset['Fare'] <= 7.91), 'Fare'] = 0
    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1
    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare']   = 2
    dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
    dataset['Fare'] = dataset['Fare'].astype(int)

train_df = train_df.drop(['FareBand'], axis=1)
combine = [train_df, test_df]
train_df.head(10)

test_df.head(10)

#model building
X_train = train_df.drop("Survived", axis=1)
Y_train = train_df["Survived"]
X_test  = test_df.drop("PassengerId", axis=1).copy()
X_train.shape, Y_train.shape, X_test.shape

# 1.Logistic Regression
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
Y_pred = logreg.predict(X_test)
acc_log = round(logreg.score(X_train, Y_train) * 100, 2)
acc_log

# =============================================================================
# Sex is highest positivie coefficient, implying as the Sex value increases (male: 0 to female: 1), the probability of Survived=1 increases the most.
# Inversely as Pclass increases, probability of Survived=1 decreases the most.
# This way Age*Class is a good feature to model as it has second highest negative correlation with Survived.
# So is Title as second highest positive correlation.
# =============================================================================
coeff_df = pd.DataFrame(train_df.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df["Correlation"] = pd.Series(logreg.coef_[0])
coeff_df.sort_values(by='Correlation', ascending=False)

#---------------------------------------------------------
#   2. Support Vector Machines
#---------------------------------------------------------
svc = SVC()
svc.fit(X_train, Y_train)
Y_pred = svc.predict(X_test)
acc_svc = round(svc.score(X_train, Y_train) * 100, 2)
acc_svc

#---------------------------------------------------------
#   3. K-nearest neighbour
#---------------------------------------------------------
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train, Y_train)
Y_pred = knn.predict(X_test)
acc_knn = round(knn.score(X_train, Y_train) * 100, 2)
acc_knn

#---------------------------------------------------------
#   4. Gaussian Naive Bayes(least score)
#---------------------------------------------------------
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
Y_pred = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_train, Y_train) * 100, 2)
acc_gaussian

#---------------------------------------------------------
#   5. Decision Tree
#---------------------------------------------------------
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
Y_pred = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(X_train, Y_train) * 100, 2)
acc_decision_tree

#---------------------------------------------------------
#   6. Random Forest(max score alongside decision tree)
#---------------------------------------------------------
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X_train, Y_train)
Y_pred = random_forest.predict(X_test)
random_forest.score(X_train, Y_train)
acc_random_forest = round(random_forest.score(X_train, Y_train) * 100, 2)
acc_random_forest

#MODEL EVALUATION
models = pd.DataFrame({
    'Model': ['Logistic Regression','Support Vector Machines', 'KNN',
               'Naive Bayes','Decision Tree','Random Forest',],
    'Score': [acc_log, acc_svc, acc_knn, acc_gaussian,
              acc_decision_tree,acc_random_forest]})
models.sort_values(by='Score', ascending=False)

#test data predictions
results = pd.DataFrame({
        "PassengerId": test_df["PassengerId"],
        "Survived": Y_pred
    })

print(results)