#  Diagonal Iteration and Reversal
class Solution:
    def findDiagonalOrder(self, matrix: List[List[int]]) -> List[int]:
        # Check for empty matrices
        if not matrix or not matrix[0]:
            return []
        # Variables to track the size of the matrix
        N, M = len(matrix), len(matrix[0])
        # The two arrays as explained in the algorithm
        result, intermediate = [], []
        # We have to go over all the elements in the first
        # row and the last column to cover all possible diagonals
        for d in range(N + M - 1):
            # Clear the intermediate array everytime we start
            # to process another diagonal
            intermediate.clear()
            # We need to figure out the "head" of this diagonal
            # The elements in the first row and the last column
            # are the respective heads.
            r, c = 0 if d < M else d - M + 1, d if d < M else M - 1
            # Iterate until one of the indices goes out of scope
            # Take note of the index math to go down the diagonal
            while r < N and c > -1:
                intermediate.append(matrix[r][c])
                r += 1
                c -= 1
            # Reverse even numbered diagonals. The
            # article says we have to reverse odd
            # numbered articles but here, the numbering
            # is starting from 0 :P
            if d % 2 == 0:
                result.extend(intermediate[::-1])
            else:
                result.extend(intermediate)
        return result

# Simulation: find starting point of diagonals.
class Solution:
    def findDiagonalOrder(self, mat: List[List[int]]) -> List[int]:
        rows, cols = len(mat), len(mat[0])
        up, down, ans = deque([]),deque([]), []
        r,c = 0,0
        while rows+c+1 <= rows+cols:
            if r < rows:
                up.append((r,c))
                r+=2
            else:
                if c == 0:
                    c = r-(rows-1)
                up.append((rows-1,c))
                c += 2
        r,c = 0,1
        while cols+r+1 <= rows+cols:
            if c < cols:
                down.append((r,c))
                c+=2
            else:
                if r == 0:
                    r = c-(cols-1)
                down.append((r,cols-1))
                r += 2
        while up and down:
            upR, upC = up.popleft()
            while upR >= 0 and upC < cols:
                ans.append(mat[upR][upC])
                upR-=1
                upC+=1
            dR, dC = down.popleft()
            while dR < rows and dC >= 0:
                ans.append(mat[dR][dC])
                dR+=1
                dC-=1
        if up:
            upR, upC = up.popleft()
            while upR >= 0 and upC < cols:
                ans.append(mat[upR][upC])
                upR-=1
                upC+=1
        return ans

#


from xai.brain.wordbase.nouns._thriller import _THRILLER

#calss header
class _THRILLERS(_THRILLER, ):
	def __init__(self,):
		_THRILLER.__init__(self)
		self.name = "THRILLERS"
		self.specie = 'nouns'
		self.basic = "thriller"
		self.jsondata = {}


"""Presentation message handler."""

from .....core.oob_processor import OobMessageProcessor
from .....ledger.error import LedgerError
from .....messaging.base_handler import BaseHandler, HandlerException
from .....messaging.models.base import BaseModelError
from .....messaging.request_context import RequestContext
from .....messaging.responder import BaseResponder
from .....storage.error import StorageError
from .....utils.tracing import trace_event, get_timer

from .. import problem_report_for_record
from ..manager import PresentationManager
from ..messages.presentation import Presentation
from ..messages.presentation_problem_report import ProblemReportReason

class PresentationHandler(BaseHandler):
    """Message handler class for presentations."""

    async def handle(self, context: RequestContext, responder: BaseResponder):
        """Message handler logic for presentations.

        Args:
            context: request context
            responder: responder callback

        """
        r_time = get_timer()
        profile = context.profile
        self._logger.debug("PresentationHandler called with context %s", context)
        assert isinstance(context.message, Presentation)
        self._logger.info(
            "Received presentation message: %s",
            context.message.serialize(as_string=True),
        )

        # If connection is present it must be ready for use
        if context.connection_record and not context.connection_ready:
            raise HandlerException("Connection used for presentation not ready")

        # Find associated oob record. If the presentation request was created as an oob
        # attachment the presentation exchange record won't have a connection id (yet)
        oob_processor = context.inject(OobMessageProcessor)
        oob_record = await oob_processor.find_oob_record_for_inbound_message(context)

        # Normally we would do a check here that there is either a connection or
        # an associated oob record. However as present proof supported receiving
        # presentation without oob record or connection record
        # (aip-1 style connectionless) we can't perform this check here

        presentation_manager = PresentationManager(profile)

        presentation_exchange_record = await presentation_manager.receive_presentation(
            context.message, context.connection_record, oob_record
        )  # mgr saves record state null if need be and possible

        r_time = trace_event(
            context.settings,
            context.message,
            outcome="PresentationHandler.handle.END",
            perf_counter=r_time,
        )

        # Automatically move to next state if flag is set
        if (
            presentation_exchange_record
            and presentation_exchange_record.auto_verify
            or context.settings.get("debug.auto_verify_presentation")
        ):
            try:
                await presentation_manager.verify_presentation(
                    presentation_exchange_record, responder
                )
            except (BaseModelError, LedgerError, StorageError) as err:
                self._logger.exception(err)
                if presentation_exchange_record:
                    async with profile.session() as session:
                        await presentation_exchange_record.save_error_state(
                            session,
                            reason=err.roll_up,  # us: be specific
                        )
                    await responder.send_reply(
                        problem_report_for_record(
                            presentation_exchange_record,
                            ProblemReportReason.ABANDONED.value,  # them: be vague
                        )
                    )

            trace_event(
                context.settings,
                presentation_exchange_record,
                outcome="PresentationHandler.handle.VERIFY",
                perf_counter=r_time,
            )


import os
import glob
import pandas as pd
import csv
import numpy as np

import os.path
import sys
import os,sys,inspect

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

import userScript

currentModule = "appendRecords"

def main(x):
	workflowNumber =x
	print(x)

	if workflowNumber == "1":
		orderOfModules = userScript.orderOfModules1
		inputDataset = userScript.inputDataset1
		outputLocation = userScript.outputLocation1
		years = userScript.generateRecordsYears1
	elif workflowNumber == "2":
		orderOfModules = userScript.orderOfModules2
		inputDataset = userScript.inputDataset2
		outputLocation = userScript.outputLocation2
		years = userScript.generateRecordsYears2

	df = pd.DataFrame()
	for i in range(len(orderOfModules)):
		#print(orderOfModules[i])
		if currentModule == orderOfModules[i]:
			if i == 0:
				df = pd.read_csv(inputDataset)
				break
			else:
				previousModule = orderOfModules[i-1]
				df = pd.read_csv(outputLocation + previousModule + ".csv")
				break

	outputDataset = outputLocation + currentModule + ".csv"
	# GENERATE COMPARATIVE DATAFRAME CONTAINING RIOT INFORMATION

	# locations in dataframe
	dfYear=df["year"]
	dfMonth=df["month"]
	dfDate=df["date"]
	dfCountry=df["country"]

	# SEPERATE DATAFRAMES BY COUNTRY
	uniqueCountries=df["country"].unique()

	# determine unique countries; used to make seperate dataframe per country
	for i in df["country"].unique():
		dfCountry=pd.DataFrame()
		k=0
		for j in df["country"]:
			# query to check whether the country matches between the dataframe and row
			if j == i:
				l=df.loc[k][:]
				# matches appended to country dataframe
				dfCountry=dfCountry.append(l)
			k=k+1
		# write each country dataframe to csv file. (in seperate folder)
		dfCountry.to_csv(outputLocation+"countryDF/dfCountry"+i+".csv", sep=',', encoding='utf-8', index=False, header=True)

	# INSERT NEW RECORD FOR EVERY DAY IN THE YEAR

	loc = outputLocation+"countryDF/"

	list_of_df = []

	# run loop for all files in previously created dataframe folder
	for f in os.listdir(loc):
		if f.endswith(".csv"):
			df = pd.read_csv(loc+f,  sep = ',')
			# append dataframe to a list
			list_of_df.append(df)

	ThirtyDays = [4,6,9,11]
	ThirtyOneDays = [1,3,5,7,8,10,12]
	TwentyEightDays = [2]

	# create records with zero values for unrest for all days, months and years
	for df in list_of_df:
		# get country name for dataframe from the first row of df
		countryName = df.loc[1]["country"]

		for y in years:
			for m in range (1,13):
				#print(m)
				if m in ThirtyDays:
					for d in range (1,31):
						df = df.append({'country':countryName, 'date':d, 'label':0, 'month':m, 'year':y}, ignore_index=True)

				if m in TwentyEightDays:
					for d in range (1,29):
						df = df.append({'country':countryName, 'date':d, 'label':0, 'month':m, 'year':y}, ignore_index=True)

				if m in ThirtyOneDays:
					for d in range (1,32):
						df = df.append({'country':countryName, 'date':d, 'label':0, 'month':m, 'year':y}, ignore_index=True)

		df.to_csv(outputLocation+"filledDF/dfFilledCountry"+countryName+".csv", sep=',', encoding='utf-8', index=False, header=True)

	# REMOVE DUPLICATE ROWS

	loc2 = outputLocation+"filledDF/"

	list_of_filled_df = []

	# run loop for all files in previously created dataframe folder
	for f in os.listdir(loc2):
		if f.endswith(".csv"):
			df = pd.read_csv(loc2+f,  sep = ',')
			# append dataframe to a list
			list_of_filled_df.append(df)

	# handle duplicate zero records
	for df in list_of_filled_df:
		# get country name for dataframe from the first row of df
		countryName = df.loc[1]["country"]

		# Select all duplicate rows based on multiple columns
		# leave the first record (showing riot) and delete the next
	# For the subset argument, specify the first n-1 columns
	df = df[['year', 'month', 'date', 'country','label']]
	df = df.drop_duplicates(subset=df.columns[:-1], keep='first')
	df.to_csv(outputLocation+"removeDuplicateDF/dfRemoveDuplicate"+countryName+".csv", sep=',', encoding='utf-8', index=False, header=False)

	# COMBINE ALL DATAFRAMES

	loc4 = outputLocation+"removeDuplicateDF/"
	os.chdir(loc4)

	allFiles = os.listdir(loc4)

	# CSV file selection

	selectedFiles = []

	for filename in allFiles:
		isFile = os.path.isfile(filename)
		if(isFile != False):
			#print(isDirectory)
			selectedFiles.append(filename)

	# Create new CSV file to write all CSV files generated from previous step
	with open(outputLocation+currentModule+".csv", "w", newline='', encoding="utf8") as outcsv:
		writer = csv.writer(outcsv, delimiter=',')

		# write the header
		writer.writerow(["Year", "Month", "Date", "ActorGeo_CountryCode", "Indicator"])

		# write the actual content line by line
		for filename in selectedFiles:
			with open(filename, 'r', newline='', encoding="utf8") as incsv:
				reader = csv.reader(incsv, delimiter=',')
				writer.writerows(row for row in reader)

	print("Module Completed: Create labelled data record for all days in range")
	return years, uniqueCountries

if __name__ == '__main__':
	years = main(sys.argv[1])
	#print(years)


import time
import math
import os
import sys
import numpy as np
import pandas as pd
from pandas import read_excel
# import xlrd
# import matplotlib.pyplot as plt

#todo 全改寫成array 看看會不會比較快 改寫前最快 3.8秒 改寫後0.6秒!

class portfolio:
    """portfolio class, contain stock and cash info\n
    """
    def __init__(self, stk_unit=0, stk_ucost=0, cash_value=100000):
        self.stk_unit = stk_unit
        self.stk_ucost = stk_ucost
        self.cash_value = cash_value
        self.stk_pv = self.stk_ucost
        print('portoflio init, starting cash :', cash_value)

    def nav(self):
        """return net asset value,
         equal to stk_pv*stk_unit + cash
        """
        return (self.stk_pv*self.stk_unit)+self.cash_value
    def tcost(self):
        """return total cost of stock in this portfolio,
         equals to stk_ucost*stk_unit
        """
        return self.stk_ucost*self.stk_unit

    def reset(self, init_cash=100000):
        """reset portfolio to an initial state,
         with 0 stock and 100000 cash in portfolio\n
        arguments :reset(init_cash=100000)
        """
        self.stk_unit = 0
        self.stk_ucost = 0
        self.stk_pv = 0
        self.cash_value = init_cash
        print('reset portfolio, cash =', init_cash)

class Environment:
    """Environment class, contain market data and tradeing-related ratios\n
    default arguments:(dataloc='./data/testdata.xls', trade_cost=0.001425, income_tax=0.003)\n

    object: trade_cost, income_tax, mktdata, maxstep\n
    trade_cost: ratio of trading cost, default equals to 1.425/1000\n
    income_tax: ratio of income tax, usually occur when selling stock, dafault is 0.3%\n
    stepcnt: step that is taking in the simulation environment\n
    mktdata: a tuple object, contain market data form dataloc\n
    default data structure is\n
    [(Date,Open,High,Low,Close,Volume,Adj Close,action,value)]\n
    """
    def __init__(self, dataloc='./data/testdata.xls', trade_cost=0.001425, income_tax=0.003):
        print('Env init, read testdata..')
        self.trade_cost = trade_cost
        #抓資料用pandas的功能比較方便 後面的計算好像用不上 pandas的dataframe合併的速度太慢 不好用
        df1 = read_excel(dataloc, sheetname=0, index_col=0).round(3)
        df2 = read_excel(dataloc, sheetname=1, index_col=0).round(6)
        self.mktdata = pd.concat([df1, df2], axis=1, join_axes=[df1.index]).fillna(0).to_records()
        #[(Date,Open,High,Low,Close,Volume,Adj Close,action,value)]
        self.maxstep = len(self.mktdata)
        print('sucessful read', self.maxstep, 'days of data, data range:', \
        self.mktdata[0][0].strftime('%Y-%m-%d'), '-', \
        self.mktdata[self.maxstep-1][0].strftime('%Y-%m-%d'))
        self.income_tax = income_tax

class trader:
    """trader class, decide a trade order is done succesful or fail,
    and calculate the nav by the info from trade order\n
    default arguments: (envi, cash_rsv=0.1)\n
    envi(required): must be an Environment() object\n
    cash_rsv: the cash reservation ratio that the trader will left in the portfolio use for trade

    objects: order_result, envi, cash_rsv\n
    method:o_result

    """
    #N日下的交易決策，用N+1日的市場資料資料判斷可不可能成交
    # envi要拋入整個Env!
    def __init__(self, envi, cash_rsv=0.01):
        self.envi = envi
        #買進股票要有現金+手續費 留0.01現金
        self.cash_rsv = cash_rsv
        print('trader init...')

    #交易指示用的port要用class portfolio做出來
    def trading(self, trade_order, trade_p, trade_v, mkt_day, port):
        """
        def o_result(self, trade_order, trade_p, trade_v, mkt_day, port)
        """

        #0表示未成交 1表示成交
        stepresult = 0
        #先檢查下的單有沒有問題
        #沒有交易 就不檢查惹 回傳1 不交易=>成交
        if trade_order == 0:
            # print('1')
            stepresult = 1
            return stepresult
        #太少量不交易
        if trade_v < 0:
            # print('2')
            return stepresult
        if trade_v < 1000:
            # print('3')
            return stepresult
        if trade_p < 0:
            # print('4')
            return stepresult

        #抓資料
        tdayh = mkt_day[2]
        tdayl = mkt_day[3]

        #檢查下的單能不能用庫存執行
        #賣出要有股票庫存
        if trade_order == -1 and port.stk_unit - trade_v < 0:
            # print('5')
            return stepresult
        #買進股票要有現金+手續費 留一成現金
        if trade_order == 1 and port.cash_value/(trade_p*trade_v*(1+self.envi.trade_cost)) < self.cash_rsv:
            # print('6')
            return stepresult
        #賣出價不能比當日最高價高 買入價不能比最低價低
        if (tdayh < trade_p and trade_order == -1) or (tdayl > trade_p and trade_order == 1):
            # print('7')
            return stepresult
        #都通過就完成交易 填入股票庫存現金庫存

        if trade_order == -1:
            port.cash_value += np.floor((trade_p*trade_v/(1+self.envi.trade_cost)*(1-self.envi.income_tax)))
        if trade_order == 1:
            port.cash_value -= np.floor((trade_p*trade_v*(1+self.envi.trade_cost)))

        if trade_order == -1 and port.stk_unit == trade_p*trade_v:
            port.stk_ucost = 0
            port.stk_unit = 0
        else:
            if trade_order == -1:
                port.ucost = port.stk_ucost
                port.stk_unit = port.stk_unit-np.floor(trade_v)
            if trade_order == 1:
                port.ucost = np.floor(port.stk_ucost*port.stk_unit + trade_p*trade_v)/np.floor(trade_v+port.stk_unit)
                port.stk_unit = port.stk_unit+np.floor(trade_v)
        stepresult = 1
        return stepresult

# def maindummy(env1, port1, trader1, step=None):
#     """main dummy simulation
#     need to use Environment() class object as env1 variable\n
#     need to use portfolio() class object as port1 variable\n
#     need to use trader() class object as trader1 variable\n
#     step is optional, decide how many step to take in the sim environment
#     """
#     test_port = port1
#     # 直接產生第一天的市場資訊 把mktdata讀進來function裡 比較快
#     mktdata = env1.mktdata
#     step_date_mkt = mktdata[0]
#     step_date_nav = test_port.nav()
#     # 直接產生第一天的交易資訊 沒交易=>成交!
#     step_o_reslt = 1
#     step_date_order = 0
#     # 直接產生第一天的結果
#     step_result = [mktdata[0][0], step_date_nav, 0, 0]
#     sim_result = [step_result]
#     # 暫存用來算sharpe ratio的資料
#     average_r = 0
#     average_r_sqr = 0
#     step_date_sharpe = 0
#     # 不管事訓練還是測試 第一天就這樣過惹 0.0
#     stepcnt = 1
#     if step == None:
#         maxstep = env1.maxstep
#     else:
#         maxstep = step
#     # 測試案例
#     # dummy策略:用前一天收盤價當下單價 每10天下單一次
#     dummytradplcy = 10
#     print('start dummp trading,', maxstep, 'steps')

#     while stepcnt < maxstep:
#         if step_date_nav <= 0:
#             print('NAV <= 0 at day', stepcnt)
#             break
#         #[(Date,Open,High,Low,Close,Volume,Adj Close,action,value)]
#         # 讀入step日的資料 移出step-1日的資料 放暫存
#         # NAV要交易後才有當日資料 mtk是當日一開始就知道了(實際上不可能先知道@@)
#         last_trade_order = step_date_order
#         last_o_reslt = step_o_reslt
#         last_step_mkt = step_date_mkt
#         step_date_mkt = mktdata[stepcnt]
#         last_date_result = step_result

#         # 輸出給AI的變數(s_N-1,a_N-1, r_N-1, s)要在這之前給齊
#         # 拿到AI給的決策之後 才產出N日的result

#         # 除權息的影響在交易前算出，在交易後加入NAV跟股票庫存
#         daily_ca = [step_date_mkt[7], step_date_mkt[8]]
#         ca_cash = 0
#         ca_stock = 0
#         # 除權息的影響在交易前算出，在交易後加入NAV跟股票庫存
#         if daily_ca[0] == 'DIVIDEND':
#             ca_cash = math.floor(test_port.stk_unit *
#                                  daily_ca[1] * (1 - env1.income_tax))
#         if daily_ca[0] == 'SPLIT':
#             ca_stock = math.floor(test_port.stk_unit * daily_ca[1])

#         if stepcnt % dummytradplcy == 1:
#             step_date_order = 1
#             step_o_reslt = trader1.o_result(step_date_order, last_step_mkt[
#                                             4], 1000, step_date_mkt, test_port)
#         else:
#             step_date_order = 0
#         port1.stk_pv = step_date_mkt[4]
#         # 計算CA對portfoilo的影響
#         if ca_cash != 0:
#             test_port.cash_value += ca_cash
#         if ca_stock != 0:
#             tmp_tcost = test_port.tcost
#             test_port.stk_unit += ca_stock
#             test_port.stk_ucost = math.floor(
#                 tmp_tcost / test_port.stk_unit * 10000) / 10000
#         step_date_nav = test_port.nav()
#         # 算to-date sharpe ratio
#         # sharpe ratio = e[x]/std[x]
#         # 算不出ratio 就直接輸出0
#         #std= (e[x]^2-e[x^2])^0.5
#         # return轉成int再轉回比較乾淨的浮點 計算速度比較快
#         daily_return = math.floor(
#             ((step_date_nav / last_date_result[1]) - 1) * 100000) / 100000

#         average_r = ((average_r * stepcnt) + daily_return) / (stepcnt + 1)
#         average_r_sqr = ((average_r_sqr * stepcnt) +
#                          daily_return**2) / (stepcnt + 1)
#         stdev_r = (average_r_sqr - average_r**2)**0.5
#         # step_date_sharpe = 0
#         if stdev_r != 0:
#             step_date_sharpe = average_r / stdev_r
#         else:
#             step_date_sharpe = 0

#         step_result = [step_date_mkt[0], step_date_nav,
#                        daily_return, step_date_sharpe]
#         # print(step_result)
#         sim_result.append(step_result)
#         stepcnt += 1
#     sim_result = pd.DataFrame(
#         columns=['date', 'NAV', 'daily return', 'sharpe to_date'], data=sim_result)
#     sim_result = sim_result.set_index(['date'])
#     sim_result = sim_result.round(
#         {'NAV': 2, 'daily return': 3, 'sharpe to_date': 6})

#     print(test_port.nav())
#     test_port.reset(init_cash=1000000)

#     #這兩行畫圖用的 import matplotlib.pyplot as plt一起
#     # reward.plot()
#     # plt.show()

# #-----------程式碼測試區 0.0
# port1 = portfolio(cash_value=1000000)
# env1 = Environment()
# trader1 = trader(env1)

# counter = 0
# while counter < 100:
#     maindummy(env1, port1, trader1)
#     counter += 1
# # maindummy(env1, port1, trader1)


from django.contrib.auth import User
from django.db import models
from django.utils.translation import ugettext_lazy as __

from mezzanine.pages.models import RichTextPage

class Project(RichTextPage, AdminTumbMixin, ):
    image = models.ImageField(
        verbose_name=__("Image"),
        upload_to='projects/' + self.title +'/images',
        height_field=700,
        width_field=350,
        help_text=__("The image used for the project's page"))

    icon = models.ImageField(
        verbose_name=__("Icon"),
        upload_to='projects/' + self.title + '/images',
        height_field=60,
        width_field=60,
        help_text=__("An icon that represents the project"))

    site_url = models.URLField(
        verbose_name=__("Project Website URL"),
        help_text=__("URL pointing to the projects own site"))

    version = models.DecimalField(
        verbose_name=__("Version"),
        max_digits=5,
        decimal_places=2,
        help_text=__("The projects current version"))

    overview = models.TextField(
        help_text=__("An overview of the project"))

    class Meta:
        verbose_name = __("Project")
        verbose_name_plural = __("Projects")


from __future__ import annotations

import anndata
import multiprocessing
import numpy as np
import pandas as pd
import os
from os import PathLike
import pandas
import pydoc
import scipy.sparse
from typing import Dict, List, Union
import warnings

from sfaira.data.dataloaders.base.dataset import DatasetBase
from sfaira.data.dataloaders.base.utils import is_child
from sfaira.versions.genomes.genomes import GenomeContainer
from sfaira.consts import AdataIds, AdataIdsSfaira
from sfaira.data.utils import read_yaml

UNS_STRING_META_IN_OBS = "__obs__"

def map_fn(inputs):
    """
    Functional to load data set with predefined additional actions.

    :param inputs:
    :return: None if function ran, error report otherwise
    """
    ds, load_raw, allow_caching, kwargs, func, kwargs_func = \
        inputs
    try:
        ds.load(load_raw=load_raw, allow_caching=allow_caching, **kwargs)
        if func is not None:
            x = func(ds, **kwargs_func)
            ds.clear()
            return x
        else:
            return None
    except (FileNotFoundError, OSError) as e:
        return ds.id, e,

def merge_uns_from_list(adata_ls):
    """
    Merge .uns from list of adata objects.

    Merges values for innert join of keys across all objects. This will retain uns streamlining.
    Keeps shared uns values for a given key across data sets as single value (not a list of 1 unique value).
    Other values are represented as a list of all unique values found.
    """
    uns_keys = [list(x.uns.keys()) for x in adata_ls]
    uns_keys_shared = set(uns_keys[0])
    for x in uns_keys[1:]:
        uns_keys_shared = uns_keys_shared.intersection(set(x))
    uns_keys_shared = list(uns_keys_shared)
    uns = {}
    for k in uns_keys_shared:
        uns_k = []
        for y in adata_ls:
            x = y.uns[k]
            if isinstance(x, list):
                pass
            elif isinstance(x, tuple):
                x = list(x)
            elif isinstance(x, np.ndarray):
                x = x.tolist()
            else:
                x = [x]
            uns_k.extend(x)
        uns_k = np.sort(np.unique(uns_k)).tolist()
        if len(uns_k) == 1:
            uns_k = uns_k[0]
        uns[k] = uns_k
    return uns

load_doc = \
    """
    :param remove_gene_version: Remove gene version string from ENSEMBL ID so that different versions in different data sets are superimposed.
    :param match_to_reference: Reference genomes name or False to keep original feature space.
    :param load_raw: Loads unprocessed version of data if available in data loader.
    :param allow_caching: Whether to allow method to cache adata object for faster re-loading.
    """

class DatasetGroup:
    """
    Container class that co-manages multiple data sets, removing need to call Dataset() methods directly through
    wrapping them.

    Example:

    #query loaders lung
    #from sfaira.dev.data.loaders.lung import DatasetGroupLung as DatasetGroup
    #dsg_humanlung = DatasetGroupHuman(path='path/to/data')
    #dsg_humanlung.load_all(match_to_reference='Homo_sapiens_GRCh38_97')
    #dsg_humanlung[some_id]
    #dsg_humanlung.adata
    """
    datasets: Dict[str, DatasetBase]
    _collection_id: str

    def __init__(self, datasets: dict, collection_id: str = "default"):
        self._adata_ids = AdataIdsSfaira()
        self.datasets = datasets
        self._collection_id = collection_id

    @property
    def _unknown_celltype_identifiers(self):
        return np.unqiue(np.concatenate([v._unknown_celltype_identifiers for _, v in self.datasets.items()]))

    def load(
            self,
            annotated_only: bool = False,
            load_raw: bool = False,
            allow_caching: bool = True,
            processes: int = 1,
            func=None,
            kwargs_func: Union[None, dict] = None,
            verbose: int = 0,
            **kwargs
    ):
        """
        Load all datasets in group (option for temporary loading).

        Note: This method automatically subsets to the group to the data sets for which input files were found.

        This method also allows temporarily loading data sets to execute function on loaded data sets (supply func).
        In this setting, datasets are removed from memory after the function has been executed.

        :param annotated_only:
        :param load_raw:
        :param allow_caching:
        :param processes: Processes to parallelise loading over. Uses python multiprocessing if > 1, for loop otherwise.
        :param func: Function to run on loaded datasets. map_fun should only take one argument, which is a Dataset
            instance. The return can be empty:

                def func(dataset, **kwargs_func):
                    # code manipulating dataset and generating output x.
                    return x
        :param kwargs_func: Kwargs of func.
        :param verbose: Verbosity of description of loading failure.

                - 0: no indication of failure
                - 1: indication of which data set failed in warning
                - 2: 1 with error report in warning
                - 3: reportin as in 2 but aborts with OSError
        """
        args = [
            load_raw,
            allow_caching,
            kwargs,
            func,
            kwargs_func
        ]

        if processes > 1 and len(self.datasets.items()) > 1:  # multiprocessing parallelisation
            print(f"using python multiprocessing (processes={processes}), "
                  f"for easier debugging revert to sequential execution (processes=1)")
            with multiprocessing.Pool(processes=processes) as pool:
                res = pool.starmap(map_fn, [
                    (tuple([v] + args),)
                    for k, v in self.datasets.items() if v.annotated or not annotated_only
                ])
            # Clear data sets that were not successfully loaded because of missing data:
            for x in res:
                if x is not None:
                    print(x[1])
                    del self.datasets[x[0]]
        else:  # for loop
            datasets_to_remove = []
            for k, v in self.datasets.items():
                print(f"loading {k}")
                x = map_fn(tuple([v] + args))
                # Clear data sets that were not successfully loaded because of missing data:
                if x is not None:
                    if verbose == 1:
                        warnings.warn(f"data set {k} not loaded")
                    if verbose == 2:
                        warnings.warn(f"data set {k} not loaded\nin data set {x[0]}: {x[1]}")
                    if verbose == 3:
                        raise OSError(f"data set {k} not loaded\nin data set {x[0]}: {x[1]}")
                    datasets_to_remove.append(k)
            for k in datasets_to_remove:
                del self.datasets[k]

    load.__doc__ += load_doc

    def streamline_metadata(
            self,
            schema: str = "sfaira",
            clean_obs: bool = True,
            clean_var: bool = True,
            clean_uns: bool = True,
            clean_obs_names: bool = True,
            keep_orginal_obs: bool = False,
            keep_symbol_obs: bool = True,
            keep_id_obs: bool = True,
    ):
        """
        Streamline the adata instance in each data set to output format.
        Output format are saved in ADATA_FIELDS* classes.

        :param schema: Export format.
            - "sfaira"
            - "cellxgene"
        :param clean_obs: Whether to delete non-streamlined fields in .obs, .obsm and .obsp.
        :param clean_var: Whether to delete non-streamlined fields in .var, .varm and .varp.
        :param clean_uns: Whether to delete non-streamlined fields in .uns.
        :param clean_obs_names: Whether to replace obs_names with a string comprised of dataset id and an increasing integer.
        :param clean_obs_names: Whether to replace obs_names with a string comprised of dataset id and an increasing
            integer.
        :param keep_orginal_obs: For ontology-constrained .obs columns, whether to keep a column with original
            annotation.
        :param keep_symbol_obs: For ontology-constrained .obs columns, whether to keep a column with ontology symbol
            annotation.
        :param keep_id_obs: For ontology-constrained .obs columns, whether to keep a column with ontology ID annotation.
        :return:
        """
        for x in self.ids:
            self.datasets[x].streamline_metadata(
                schema=schema,
                clean_obs=clean_obs,
                clean_var=clean_var,
                clean_uns=clean_uns,
                clean_obs_names=clean_obs_names,
                keep_orginal_obs=keep_orginal_obs,
                keep_symbol_obs=keep_symbol_obs,
                keep_id_obs=keep_id_obs,
            )

    def streamline_features(
            self,
            match_to_reference: Union[str, Dict[str, str], None],
            remove_gene_version: bool = True,
            subset_genes_to_type: Union[None, str, List[str]] = None,
    ):
        """
        Subset and sort genes to genes defined in an assembly or genes of a particular type, such as protein coding.
        :param match_to_reference: Which annotation to map the feature space to. Can be:
            - str: Provide the name of the annotation in the format Organism.Assembly.Release
            - dict: Mapping of organism to name of the annotation (see str format). Chooses annotation for each data set
                based on organism annotation.
        :param remove_gene_version: Whether to remove the version number after the colon sometimes found in ensembl gene ids.
        :param subset_genes_to_type: Type(s) to subset to. Can be a single type or a list of types or None. Types can be:
            - None: All genes in assembly.
            - "protein_coding": All protein coding genes in assembly.
        """
        for x in self.ids:
            self.datasets[x].streamline_features(
                match_to_reference=match_to_reference,
                remove_gene_version=remove_gene_version,
                subset_genes_to_type=subset_genes_to_type,
            )

    def collapse_counts(self):
        """
        Collapse count matrix along duplicated index.
        """
        for x in self.ids:
            self.datasets[x].collapse_counts()

    def write_distributed_store(
            self,
            dir_cache: Union[str, os.PathLike],
            store_format: str = "dao",
            dense: bool = False,
            compression_kwargs: dict = {},
            chunks: Union[int, None] = None,
    ):
        """
        Write data set into a format that allows distributed access to data set on disk.

        Stores are useful for distributed access to data sets, in many settings this requires some streamlining of the
        data sets that are accessed. Use .streamline_* before calling this method to streamline the data sets.
        This method writes a separate file for each data set in this object.

        :param dir_cache: Directory to write cache in.
        :param store_format: Disk format for objects in cache. Recommended is "dao".

            - "h5ad": Allows access via backed .h5ad.
                Note on compression: .h5ad supports sparse data with is a good compression that gives fast row-wise
                    access if the files are csr, so further compression potentially not necessary.
            - "dao": Distributed access optimised format, recommended for batched access in optimisation, for example.
        :param dense: Whether to write sparse or dense store, this will be homogenously enforced.
        :param compression_kwargs: Compression key word arguments to give to h5py or zarr
            For store_format=="h5ad", see also anndata.AnnData.write_h5ad:
                - compression,
                - compression_opts.
            For store_format=="dao", see also sfaira.data.write_dao which relays kwargs to
            zarr.hierarchy.create_dataset:
                - compressor
                - overwrite
                - order
                and others.
        :param chunks: Observation axes of chunk size of zarr array, see anndata.AnnData.write_zarr documentation.
            Only relevant for store=="dao". The feature dimension of the chunks is always is the full feature space.
            Uses zarr default chunking across both axes if None.
        """
        for _, v in self.datasets.items():
            v.write_distributed_store(dir_cache=dir_cache, store_format=store_format, dense=dense,
                                      compression_kwargs=compression_kwargs, chunks=chunks)

    def write_backed(
            self,
            adata_backed: anndata.AnnData,
            genome: str,
            idx: List[np.ndarray],
            annotated_only: bool = False,
            load_raw: bool = False,
            allow_caching: bool = True,
    ):
        """
        Loads data set group into slice of backed anndata object.

        Subsets self.datasets to the data sets that were found. Note that feature space is automatically formatted as
        this is necessary for concatenation.

        :param adata_backed: Anndata instance to load into.
        :param genome: Genome container target genomes loaded.
        :param idx: Indices in adata_backed to write observations to. This can be used to immediately create a
            shuffled object. This has to be a list of the length of self.data, one index array for each dataset.
        :param annotated_only:
        :param load_raw: See .load().
        :param allow_caching: See .load().
        :return: New row index for next element to be written into backed anndata.
        """
        i = 0
        for x in self.ids:
            # if this is for celltype prediction, only load the data with have celltype annotation
            try:
                if self.datasets[x].annotated or not annotated_only:
                    self.datasets[x].write_backed(
                        adata_backed=adata_backed,
                        genome=genome,
                        idx=idx[i],
                        load_raw=load_raw,
                        allow_caching=allow_caching
                    )
                    i += 1
            except FileNotFoundError:
                del self.datasets[x]

    def write_ontology_class_map(
            self,
            fn,
            protected_writing: bool = True,
            **kwargs
    ):
        """
        Write cell type maps of free text cell types to ontology classes.

        :param fn: File name of tsv to write class maps to.
        :param protected_writing: Only write if file was not already found.
        """
        tab = []
        for k, v in self.datasets.items():
            if v.annotated:
                labels_original = np.sort(np.unique(np.concatenate([
                    v.adata.obs[v.cell_type_original_obs_key].values
                ])))
                tab.append(v.celltypes_universe.prepare_celltype_map_tab(
                    source=labels_original,
                    match_only=False,
                    anatomical_constraint=v.organ,
                    include_synonyms=True,
                    omit_list=v._unknown_celltype_identifiers,
                    **kwargs
                ))
        if len(tab) == 0:
            warnings.warn("attempted to write ontology classmaps for group without annotated data sets")
        else:
            tab = pandas.concat(tab, axis=0)
            # Take out columns with the same source:
            tab = tab.loc[[x not in tab.iloc[:i, 0].values for i, x in enumerate(tab.iloc[:, 0].values)], :].copy()
            tab = tab.sort_values(self._adata_ids.classmap_source_key)
            if not os.path.exists(fn) or not protected_writing:
                tab.to_csv(fn, index=False, sep="\t")

    def download(self, **kwargs):
        for _, v in self.datasets.items():
            v.download(**kwargs)

    @property
    def ids(self):
        return list(self.datasets.keys())

    @property
    def collection_id(self):
        return self._collection_id

    @property
    def adata_ls(self):
        adata_ls = []
        for i in self.ids:
            if self.datasets[i] is not None:
                if self.datasets[i].adata is not None:
                    adata_ls.append(self.datasets[i].adata)
        return adata_ls

    @property
    def adata(self):
        adata_ls = self.adata_ls
        if not adata_ls:
            return None
        if len(adata_ls) == 1:
            for i in self.ids:
                if self.datasets[i] is not None:
                    if self.datasets[i].adata is not None:
                        ds_id = i
            adata_concat = adata_ls[0]
            adata_concat.obs[self._adata_ids.dataset] = ds_id
        else:
            # Check that all individual adata objects in linked Dataset instances have identicall streamlined features and metadata
            match_ref_list = []
            rm_gene_ver_list = []
            gene_type_list = []
            for d_id in self.ids:
                if self.datasets[d_id].adata is not None:
                    assert self.datasets[d_id].mapped_features, f"Dataset {d_id} does not seem to have a streamlined " \
                                                                f"featurespace. To obtain an adata object from this " \
                                                                f"DatasetGroup, all contained Datasets need to have a " \
                                                                f"streamlined featurespace. Run .streamline_features()" \
                                                                f" first."
                    assert self.datasets[d_id].streamlined_meta, f"Dataset {d_id} does not seem to have streamlined " \
                                                                 f"metadata. To obtain an adata object from this " \
                                                                 f"DatasetGroup, all contained Datasets need to have " \
                                                                 f"streamlined metadata. Run .streamline_metadata() first."
                    match_ref_list.append(self.datasets[d_id].mapped_features)
                    rm_gene_ver_list.append(self.datasets[d_id].remove_gene_version)
                    gene_type_list.append(self.datasets[d_id].subset_gene_type)
            assert len(set(match_ref_list)) == 1, \
                "Not all datasets in this group had their features matched to the same reference (argument " \
                "'match_to_reference' of method .streamline_features())." \
                "This is however a prerequisite for creating a combined adata object."
            assert len(set(rm_gene_ver_list)) == 1, \
                "Not all datasets in this group have had their gene version removed (argument 'remove_gene_version' of " \
                "method .streamline_features()). This is however a prerequisite for creating a combined adata object."
            assert len(set(gene_type_list)) == 1, \
                "Not all datasets in this group had their featurespace subsetted to the same gene type (argument " \
                "'subset_gene_type' of method .streamline_features()). This is however a prerequisite for creating a " \
                "combined adata object."

            var_original = adata_ls[0].var.copy()
            for a in adata_ls:
                a.var_names_make_unique()
            # TODO: need to keep this? -> yes, still catching errors here (March 2020)
            # Fix for loading bug: sometime concatenating sparse matrices fails the first time but works on second try.
            try:
                adata_concat = adata_ls[0].concatenate(
                    *adata_ls[1:],
                    join="outer",
                    batch_key=self._adata_ids.dataset,
                    batch_categories=[i for i in self.ids if self.datasets[i].adata is not None],
                    index_unique=None
                )
            except ValueError:
                adata_concat = adata_ls[0].concatenate(
                    *adata_ls[1:],
                    join="outer",
                    batch_key=self._adata_ids.dataset,
                    batch_categories=[i for i in self.ids if self.datasets[i].adata is not None],
                    index_unique=None
                )
            adata_concat.var = var_original
            adata_concat.uns = merge_uns_from_list(adata_ls)
            adata_concat.uns[self._adata_ids.mapped_features] = match_ref_list[0]

        return adata_concat

    def obs_concat(self, keys: Union[list, None] = None):
        """
        Returns concatenation of all .obs.

        Uses union of all keys if keys is not provided.

        :param keys:
        :return:
        """
        if keys is None:
            keys = np.unique(np.concatenate([list(x.obs.columns) for x in self.adata_ls]))
        obs_concat = pandas.concat([pandas.DataFrame(dict(
            [
                (k, self.datasets[x].adata.obs[k]) if k in self.datasets[x].adata.obs.columns
                else (k, ["nan" for _ in range(self.datasets[x].adata.obs.shape[0])])
                for k in keys
            ] + [(self._adata_ids.dataset, [x for _ in range(self.datasets[x].adata.obs.shape[0])])]
        )) for x in self.ids if self.datasets[x].adata is not None])
        return obs_concat

    def ncells_bydataset(self, annotated_only: bool = False) -> np.ndarray:
        cells = []
        for x in self.ids:
            # if this is for celltype prediction, only load the data with have celltype annotation
            try:
                if self.datasets[x].annotated or not annotated_only:
                    cells.append(self.datasets[x].ncells)
            except FileNotFoundError:
                del self.datasets[x]
        return np.asarray(cells)

    def ncells(self, annotated_only: bool = False):
        cells = self.ncells_bydataset(annotated_only=annotated_only)
        return np.sum(cells)

    @property
    def ontology_celltypes(self):
        organism = np.unique([v.organism for _, v in self.datasets.items()])
        if len(organism) > 1:
            # ToDo: think about whether this should be handled differently.
            warnings.warn("found more than one organism in group, this could cause problems with using a joined cell "
                          "type ontology. Using only the ontology of the first data set in the group.")
        return self.datasets[self.ids[0]].ontology_container_sfaira.cell_type

    def project_celltypes_to_ontology(self, adata_fields: Union[AdataIds, None] = None, copy=False):
        """
        Project free text cell type names to ontology based on mapping table.
        :return:
        """
        for _, v in self.datasets.items():
            v.project_free_to_ontology(adata_fields=adata_fields, copy=copy)

    def subset(self, key, values: Union[list, tuple, np.ndarray]):
        """
        Subset list of adata objects based on sample-wise properties.

        These keys are properties that are available in lazy model.
        Subsetting happens on .datasets.

        :param key: Property to subset by.
        :param values: Classes to overlap to. Return if elements match any of these classes.
        :return:
        """
        ids_del = []
        if isinstance(values, np.ndarray):
            values = values.tolist()
        if isinstance(values, tuple):
            values = list(values)
        if not isinstance(values, list):
            values = [values]
        for x in self.ids:
            try:
                values_found = getattr(self.datasets[x], key)
            except AttributeError:
                raise ValueError(f"{key} not a valid property of data set object")
            try:
                ontology = getattr(self.datasets[x].ontology_container_sfaira, key)
            except AttributeError:
                raise ValueError(f"{key} not a valid property of ontology_container object")
            if values_found is None:
                # Delete entries which do not have this meta data item annotated.
                ids_del.append(x)
            else:
                if not isinstance(values_found, list):
                    values_found = [values_found]
                if not np.any([
                    np.any([
                        is_child(query=y, ontology=ontology, ontology_parent=z)
                        for z in values
                    ]) for y in values_found
                ]):
                    # Delete entries which a non-matching meta data value associated with this item.
                    ids_del.append(x)
        for x in ids_del:
            del self.datasets[x]

    def subset_cells(self, key, values: Union[str, List[str]]):
        """
        Subset list of adata objects based on cell-wise properties.

        These keys are properties that are not available in lazy model and require loading first because the
        subsetting works on the cell-level: .adata are maintained but reduced to matches.

        :param key: Property to subset by. Options:

            - "assay_differentiation" points to self.assay_differentiation_obs_key
            - "assay_sc" points to self.assay_sc_obs_key
            - "assay_type_differentiation" points to self.assay_type_differentiation_obs_key
            - "cell_line" points to self.cell_line
            - "cell_type" points to self.cell_type_obs_key
            - "developmental_stage" points to self.developmental_stage_obs_key
            - "ethnicity" points to self.ethnicity_obs_key
            - "organ" points to self.organ_obs_key
            - "organism" points to self.organism_obs_key
            - "sample_source" points to self.sample_source_obs_key
            - "sex" points to self.sex_obs_key
            - "state_exact" points to self.state_exact_obs_key
        :param values: Classes to overlap to.
        :return:
        """
        for x in self.ids:
            self.datasets[x].subset_cells(key=key, values=values)
            if self.datasets[x].ncells == 0:  # No observations (cells) left.
                del self.datasets[x]

    @property
    def additional_annotation_key(self) -> Dict[str, Union[None, str]]:
        """"
        Return dictionary of additional_annotation_key for each data set with ids as keys.
        """
        return dict([
            (k, self.datasets[k].additional_annotation_key)
            for k, v in self.datasets.items()
        ])

    @additional_annotation_key.setter
    def additional_annotation_key(self, x: Dict[str, Union[None, str]]):
        """
        Allows setting of additional_annotation_key in a subset of datasets identifed by keys in x.

        :param x: Dictionary with data set ids in keys and new _additional_annotation_key values to be setted in values.
            Note that you can either add  or change secondary annotation by setting a value to a string or remove it
            by setting a value to None.
        :return:
        """
        for k, v in x.items():
            self.datasets[k].additional_annotation_key = v

    @property
    def doi(self) -> List[str]:
        """
        Propagates DOI annotation from contained datasets.
        """
        dois = []
        for _, v in self.datasets.items():
            vdoi = v.doi_journal
            if isinstance(vdoi, str):
                vdoi = [vdoi]
            dois.extend(vdoi)
        return np.sort(np.unique(dois)).tolist()

    @property
    def supplier(self) -> List[str]:
        """
        Propagates supplier annotation from contained datasets.
        """
        supplier = [v.supplier for _, v in self.datasets.items()]
        return np.sort(np.unique(supplier)).tolist()

    def show_summary(self):
        for k, v in self.datasets.items():
            print(k)
            print(f"\t {(v.supplier, v.organism, v.organ, v.assay_sc, v.disease)}")

class DatasetGroupDirectoryOriented(DatasetGroup):

    _cwd: os.PathLike

    def __init__(
            self,
            file_base: str,
            data_path: Union[str, None] = None,
            meta_path: Union[str, None] = None,
            cache_path: Union[str, None] = None,
    ):
        """
        Automatically collects Datasets from python files in directory.

        Uses a pre-built DatasetGroup if this is defined in a group.py file, otherwise, the DatasetGroup is initialised
        here.

        :param file_base:
        :param data_path:
        :param meta_path:
        :param cache_path:
        """
        # Collect all data loaders from files in directory:
        datasets = []
        self._cwd = os.path.dirname(file_base)
        try:
            collection_id = str(self._cwd).split(os.sep)[-1]
            package_source = str(self._cwd).split(os.sep)[-5]
            if package_source == "sfaira":
                pass
            elif package_source == "sfaira_extension":
                package_source = "sfairae"
            else:
                raise ValueError(f"invalid package source {package_source} for {self._cwd}")
        except IndexError as e:
            raise IndexError(f"{e} for {self._cwd}")
        loader_pydoc_path_sfaira = "sfaira.data.dataloaders.loaders."
        loader_pydoc_path_sfairae = "sfaira_extension.data.dataloaders.loaders."
        loader_pydoc_path = loader_pydoc_path_sfaira if package_source == "sfaira" else loader_pydoc_path_sfairae
        if "group.py" in os.listdir(self._cwd):
            DatasetGroupFound = pydoc.locate(loader_pydoc_path + collection_id + ".group.DatasetGroup")
            dsg = DatasetGroupFound(data_path=data_path, meta_path=meta_path, cache_path=cache_path)
            datasets.extend(list(dsg.datasets.values))
        else:
            for f in os.listdir(self._cwd):
                if os.path.isfile(os.path.join(self._cwd, f)):  # only files
                    # Narrow down to data set files:
                    if f.split(".")[-1] == "py" and f.split(".")[0] not in ["__init__", "base", "group"]:
                        datasets_f = []
                        file_module = ".".join(f.split(".")[:-1])
                        DatasetFound = pydoc.locate(loader_pydoc_path + collection_id + "." + file_module + ".Dataset")
                        # Load objects from name space:
                        # - load(): Loading function that return anndata instance.
                        # - SAMPLE_FNS: File name list for DatasetBaseGroupLoadingManyFiles
                        load_func = pydoc.locate(loader_pydoc_path + collection_id + "." + file_module + ".load")
                        load_func_annotation = \
                            pydoc.locate(loader_pydoc_path + collection_id + "." + file_module + ".LOAD_ANNOTATION")
                        # Also check sfaira_extension for additional load_func_annotation:
                        if package_source != "sfairae":
                            load_func_annotation_sfairae = pydoc.locate(loader_pydoc_path_sfairae + collection_id +
                                                                        "." + file_module + ".LOAD_ANNOTATION")
                            # LOAD_ANNOTATION is a dictionary so we can use update to extend it.
                            if load_func_annotation_sfairae is not None and load_func_annotation is not None:
                                load_func_annotation.update(load_func_annotation_sfairae)
                            elif load_func_annotation_sfairae is not None and load_func_annotation is None:
                                load_func_annotation = load_func_annotation_sfairae
                        sample_fns = pydoc.locate(loader_pydoc_path + collection_id + "." + file_module +
                                                  ".SAMPLE_FNS")
                        fn_yaml = os.path.join(self._cwd, file_module + ".yaml")
                        fn_yaml = fn_yaml if os.path.exists(fn_yaml) else None
                        # Check for sample_fns in yaml:
                        if fn_yaml is not None:
                            assert os.path.exists(fn_yaml), f"did not find yaml {fn_yaml}"
                            yaml_vals = read_yaml(fn=fn_yaml)
                            if sample_fns is None and yaml_vals["meta"]["sample_fns"] is not None:
                                sample_fns = yaml_vals["meta"]["sample_fns"]
                        if sample_fns is None:
                            sample_fns = [None]
                        # Here we distinguish between class that are already defined and those that are not.
                        # The latter case arises if meta data are defined in YAMLs and _load is given as a function.
                        if DatasetFound is None:
                            for x in sample_fns:
                                datasets_f.append(
                                    DatasetBase(
                                        data_path=data_path,
                                        meta_path=meta_path,
                                        cache_path=cache_path,
                                        load_func=load_func,
                                        dict_load_func_annotation=load_func_annotation,
                                        sample_fn=x,
                                        sample_fns=sample_fns if sample_fns != [None] else None,
                                        yaml_path=fn_yaml,
                                    )
                                )
                        else:
                            for x in sample_fns:
                                datasets_f.append(
                                    DatasetFound(
                                        data_path=data_path,
                                        meta_path=meta_path,
                                        cache_path=cache_path,
                                        load_func=load_func,
                                        load_func_annotation=load_func_annotation,
                                        sample_fn=x,
                                        sample_fns=sample_fns if sample_fns != [None] else None,
                                        yaml_path=fn_yaml,
                                    )
                                )
                        # Load cell type maps:
                        for x in datasets_f:
                            x.read_ontology_class_map(fn=os.path.join(self._cwd, file_module + ".tsv"))
                        datasets.extend(datasets_f)

        keys = [x.id for x in datasets]
        super().__init__(datasets=dict(zip(keys, datasets)), collection_id=collection_id)

    def clean_ontology_class_map(self):
        """
        Finalises processed class maps of free text cell types to ontology classes.

        Checks that the assigned ontology class names appear in the ontology.
        Adds a third column with the corresponding ontology IDs into the file.

        :return:
        """
        for f in os.listdir(self._cwd):
            if os.path.isfile(os.path.join(self._cwd, f)):  # only files
                # Narrow down to data set files:
                if f.split(".")[-1] == "py" and f.split(".")[0] not in ["__init__", "base", "group"]:
                    file_module = ".".join(f.split(".")[:-1])
                    fn_map = os.path.join(self._cwd, file_module + ".tsv")
                    if os.path.exists(fn_map):
                        # Access reading and value protection mechanisms from first data set loaded in group.
                        tab = list(self.datasets.values())[0]._read_ontology_class_map(fn=fn_map)
                        # Checks that the assigned ontology class names appear in the ontology.
                        list(self.datasets.values())[0]._value_protection(
                            attr="cell_type",
                            allowed=self.ontology_celltypes,
                            attempted=[
                                x for x in np.unique(tab[self._adata_ids.classmap_target_key].values)
                                if x not in [
                                    self._adata_ids.unknown_metadata_identifier,
                                    self._adata_ids.not_a_cell_celltype_identifier
                                ]
                            ]
                        )
                        # Adds a third column with the corresponding ontology IDs into the file.
                        tab[self._adata_ids.classmap_target_id_key] = [
                            self.ontology_celltypes.convert_to_id(x)
                            if (x != self._adata_ids.unknown_metadata_identifier and
                                x != self._adata_ids.not_a_cell_celltype_identifier)
                            else self._adata_ids.unknown_metadata_identifier
                            for x in tab[self._adata_ids.classmap_target_key].values
                        ]
                        # Get writing function from any (first) data set instance:
                        k = list(self.datasets.keys())[0]
                        self.datasets[k]._write_ontology_class_map(fn=fn_map, tab=tab)

class DatasetSuperGroup:
    """
    Container for multiple DatasetGroup instances.

    Used to manipulate structured dataset collections. Primarly designed for this manipulation, convert to DatasetGroup
    via flatten() for more functionalities.
    """
    _adata: Union[None, anndata.AnnData]
    fn_backed: Union[None, PathLike]
    dataset_groups: Union[list, List[DatasetGroup], List[DatasetSuperGroup]]

    def __init__(self, dataset_groups: Union[None, List[DatasetGroup], List[DatasetSuperGroup]]):
        self._adata = None
        self.fn_backed = None
        self.set_dataset_groups(dataset_groups=dataset_groups)

        self._adata_ids = AdataIdsSfaira()

    def set_dataset_groups(self, dataset_groups: Union[DatasetGroup, DatasetSuperGroup, List[DatasetGroup],
                                                       List[DatasetSuperGroup]]):
        if isinstance(dataset_groups, DatasetGroup) or isinstance(dataset_groups, DatasetSuperGroup):
            dataset_groups = [dataset_groups]
        if len(dataset_groups) > 0:
            if isinstance(dataset_groups[0], DatasetGroup):
                self.dataset_groups = dataset_groups
            elif isinstance(dataset_groups[0], DatasetSuperGroup):
                # Decompose super groups first
                dataset_groups_proc = []
                for x in dataset_groups:
                    dataset_groups_proc.extend(x.dataset_groups)
                self.dataset_groups = dataset_groups_proc
            else:
                assert False
        else:
            self.dataset_groups = []

    def extend_dataset_groups(self, dataset_groups: Union[List[DatasetGroup], List[DatasetSuperGroup]]):
        if isinstance(dataset_groups[0], DatasetGroup):
            self.dataset_groups.extend(dataset_groups)
        elif isinstance(dataset_groups[0], DatasetSuperGroup):
            # Decompose super groups first
            dataset_groups_proc = []
            for x in dataset_groups:
                dataset_groups_proc.extend(x.datasets)
            self.dataset_groups.extend(dataset_groups_proc)
        else:
            assert False

    @property
    def ids(self):
        ids = []
        for x in self.dataset_groups:
            ids.extend(x.ids)
        return ids

    def get_gc(
            self,
            genome: str = None
    ):
        if genome.lower().startswith("homo_sapiens") or genome.lower().startswith("mus_musculus"):
            g = GenomeContainer(
                assembly=genome
            )
        else:
            raise ValueError(f"Genome {genome} not recognised. Needs to start with 'Mus_Musculus' or 'Homo_Sapiens'.")
        return g

    def ncells_bydataset(self, annotated_only: bool = False):
        """
        List of list of length of all data sets by data set group.
        :return:
        """
        return [x.ncells_bydataset(annotated_only=annotated_only) for x in self.dataset_groups]

    def ncells_bydataset_flat(self, annotated_only: bool = False):
        """
        Flattened list of length of all data sets.
        :return:
        """
        return [xx for x in self.ncells_bydataset(annotated_only=annotated_only) for xx in x]

    def ncells(self, annotated_only: bool = False):
        return np.sum(self.ncells_bydataset_flat(annotated_only=annotated_only))

    @property
    def datasets(self) -> Dict[str, DatasetBase]:
        """
        Returns DatasetGroup (rather than self = DatasetSuperGroup) containing all listed data sets.

        :return:
        """
        ds = {}
        for x in self.dataset_groups:
            for k, v in x.datasets.items():
                assert k not in ds.keys(), f"{k} was duplicated in super group, remove duplicates before flattening"
                ds[k] = v
        return ds

    def flatten(self) -> DatasetGroup:
        """
        Returns DatasetGroup (rather than self = DatasetSuperGroup) containing all listed data sets.

        :return:
        """
        ds = {}
        for x in self.dataset_groups:
            for k, v in x.datasets.items():
                assert k not in ds.keys(), f"{k} was duplicated in super group, purge duplicates before flattening"
                ds[k] = v
        return DatasetGroup(datasets=ds)

    def download(self, **kwargs):
        for x in self.dataset_groups:
            x.download(**kwargs)

    def load(
            self,
            annotated_only: bool = False,
            load_raw: bool = False,
            allow_caching: bool = True,
            processes: int = 1,
            **kwargs
    ):
        """
        Loads data set human into anndata object.

        :param annotated_only:
        :param load_raw: See .load().
        :param allow_caching: See .load().
        :param processes: Processes to parallelise loading over. Uses python multiprocessing if > 1, for loop otherwise.
            Note: parallelises loading of each dataset group, but not across groups.
        :return:
        """
        for x in self.dataset_groups:
            x.load(
                annotated_only=annotated_only,
                load_raw=load_raw,
                allow_caching=allow_caching,
                processes=processes,
                **kwargs
            )

    def streamline_features(
            self,
            match_to_reference: Union[str, Dict[str, str], None],
            remove_gene_version: bool = True,
            subset_genes_to_type: Union[None, str, List[str]] = None,
    ):
        """
        Subset and sort genes to genes defined in an assembly or genes of a particular type, such as protein coding.
        :param match_to_reference: Which annotation to map the feature space to. Can be:
            - str: Provide the name of the annotation in the format Organism.Assembly.Release
            - dict: Mapping of organism to name of the annotation (see str format). Chooses annotation for each data set
                based on organism annotation.
        :param remove_gene_version: Whether to remove the version number after the colon sometimes found in ensembl gene ids.
        :param subset_genes_to_type: Type(s) to subset to. Can be a single type or a list of types or None. Types can be:
            - None: All genes in assembly.
            - "protein_coding": All protein coding genes in assembly.
        """
        for x in self.dataset_groups:
            x.streamline_features(
                match_to_reference=match_to_reference,
                remove_gene_version=remove_gene_version,
                subset_genes_to_type=subset_genes_to_type,
            )

    def collapse_counts(self):
        """
        Collapse count matrix along duplicated index.
        """
        for x in self.dataset_groups:
            x.collapse_counts()

    @property
    def adata_ls(self):
        adata_ls = []
        for k, v in self.flatten().datasets.items():
            if v.adata is not None:
                adata_ls.append(v.adata)
        return adata_ls

    @property
    def adata(self):
        adata_ls = self.adata_ls
        if not adata_ls:
            return None
        if len(adata_ls) == 1:
            for i in self.ids:
                if self.datasets[i] is not None:
                    if self.datasets[i].adata is not None:
                        ds_id = i
            adata_concat = adata_ls[0]
            adata_concat.obs[self._adata_ids.dataset] = ds_id
        else:
            # Check that all individual adata objects in linked Dataset instances have identicall streamlined features and metadata
            match_ref_list = []
            rm_gene_ver_list = []
            gene_type_list = []
            for d_id in self.flatten().ids:
                if self.flatten().datasets[d_id].adata is not None:
                    assert self.flatten().datasets[d_id].mapped_features, f"Dataset {d_id} does not seem to have a streamlined " \
                                                                          f"featurespace. To obtain an adata object from this " \
                                                                          f"DatasetGroup, all contained Datasets need to have a " \
                                                                          f"streamlined featurespace. Run .streamline_features()" \
                                                                          f" first."
                    assert self.flatten().datasets[d_id].streamlined_meta, f"Dataset {d_id} does not seem to have streamlined " \
                                                                           f"metadata. To obtain an adata object from this " \
                                                                           f"DatasetGroup, all contained Datasets need to have " \
                                                                           f"streamlined metadata. Run .streamline_metadata() first."
                    match_ref_list.append(self.flatten().datasets[d_id].mapped_features)
                    rm_gene_ver_list.append(self.flatten().datasets[d_id].remove_gene_version)
                    gene_type_list.append(self.flatten().datasets[d_id].subset_gene_type)
            assert len(set(match_ref_list)) == 1, \
                "Not all datasets in this group had their features matched to the same reference (argument " \
                "'match_to_reference' of method .streamline_features()). This is however a prerequisite for creating a " \
                "combined adata object."
            assert len(set(rm_gene_ver_list)) == 1, \
                "Not all datasets in this group have had their gene version removed (argument 'remove_gene_version' of " \
                "method .streamline_features()). This is however a prerequisite for creating a combined adata object."
            assert len(set(gene_type_list)) == 1, \
                "Not all datasets in this group had their featurespace subsetted to the same gene type (argument " \
                "'subset_gene_type' of method .streamline_features()). This is however a prerequisite for creating a " \
                "combined adata object."

            var_original = adata_ls[0].var.copy()
            for a in adata_ls:
                a.var_names_make_unique()
            # TODO: need to keep this? -> yes, still catching errors here (March 2020)
            # Fix for loading bug: sometime concatenating sparse matrices fails the first time but works on second try.
            try:
                adata_concat = adata_ls[0].concatenate(
                    *adata_ls[1:],
                    join="outer",
                    batch_key=self._adata_ids.dataset,
                    batch_categories=[i for i in self.ids if self.flatten().datasets[i].adata is not None],
                    index_unique=None
                )
            except ValueError:
                adata_concat = adata_ls[0].concatenate(
                    *adata_ls[1:],
                    join="outer",
                    batch_key=self._adata_ids.dataset,
                    batch_categories=[i for i in self.ids if self.flatten().datasets[i].adata is not None],
                    index_unique=None
                )
            adata_concat.var = var_original
            adata_concat.uns = merge_uns_from_list(adata_ls)
            adata_concat.uns[self._adata_ids.mapped_features] = match_ref_list[0]

        return adata_concat

    def write_distributed_store(
            self,
            dir_cache: Union[str, os.PathLike],
            store_format: str = "dao",
            dense: bool = False,
            compression_kwargs: dict = {},
            chunks: Union[int, None] = None,
    ):
        """
        Write data set into a format that allows distributed access to data set on disk.

        Stores are useful for distributed access to data sets, in many settings this requires some streamlining of the
        data sets that are accessed. Use .streamline_* before calling this method to streamline the data sets.
        This method writes a separate file for each data set in this object.

        :param dir_cache: Directory to write cache in.
        :param store_format: Disk format for objects in cache. Recommended is "dao".

            - "h5ad": Allows access via backed .h5ad.
                Note on compression: .h5ad supports sparse data with is a good compression that gives fast row-wise
                    access if the files are csr, so further compression potentially not necessary.
            - "dao": Distributed access optimised format, recommended for batched access in optimisation, for example.
        :param dense: Whether to write sparse or dense store, this will be homogenously enforced.
        :param compression_kwargs: Compression key word arguments to give to h5py or zarr
            For store_format=="h5ad", see also anndata.AnnData.write_h5ad:
                - compression,
                - compression_opts.
            For store_format=="dao", see also sfaira.data.write_dao which relays kwargs to
            zarr.hierarchy.create_dataset:
                - compressor
                - overwrite
                - order
                and others.
        :param chunks: Observation axes of chunk size of zarr array, see anndata.AnnData.write_zarr documentation.
            Only relevant for store=="dao". The feature dimension of the chunks is always is the full feature space.
            Uses zarr default chunking across both axes if None.
        """
        for x in self.dataset_groups:
            x.write_distributed_store(dir_cache=dir_cache, store_format=store_format, dense=dense,
                                      compression_kwargs=compression_kwargs, chunks=chunks)

    def write_backed(
            self,
            fn_backed: PathLike,
            genome: str,
            shuffled: bool = False,
            as_dense: bool = False,
            annotated_only: bool = False,
            load_raw: bool = False,
            allow_caching: bool = True,
    ):
        """
        Loads data set human into backed anndata object.

        TODO replace streamlining in here by required call to .streamline() before.

        Example usage:

            ds = DatasetSuperGroup([...])
            ds.write_backed(
                fn_backed="...",
                target_genome="...",
                annotated_only=False
            )
            adata_backed = anndata.read(ds.fn_backed, backed='r')
            adata_slice = ad_full[idx]

        :param fn_backed: File name to save backed anndata to temporarily.
        :param genome: ID of target genomes.
        :param shuffled: Whether to shuffle data when writing to backed.
        :param as_dense: Whether to load into dense count matrix.
        :param annotated_only:
        :param load_raw: See .load().
        :param allow_caching: See .load().
        """
        if shuffled and not as_dense:
            raise ValueError("cannot write backed shuffled and sparse")
        scatter_update = shuffled or as_dense
        self.fn_backed = fn_backed
        n_cells = self.ncells(annotated_only=annotated_only)
        gc = self.get_gc(genome=genome)
        n_genes = gc.n_var
        if scatter_update:
            self.adata = anndata.AnnData(
                scipy.sparse.csr_matrix((n_cells, n_genes), dtype=np.float32)
            )  # creates an empty anndata object with correct dimensions that can be filled with cells from data sets
        else:
            self.adata = anndata.AnnData(
                scipy.sparse.csr_matrix((0, n_genes), dtype=np.float32)
            )
        self.adata.filename = fn_backed  # setting this attribute switches this anndata to a backed object
        # Note that setting .filename automatically redefines .X as dense, so we have to redefine it as sparse:
        if not as_dense:
            X = scipy.sparse.csr_matrix(self.adata.X)  # redefines this backed anndata as sparse
            X.indices = X.indices.astype(np.int64)
            X.indptr = X.indptr.astype(np.int64)
            self.adata.X = X
        keys = [
            self._adata_ids.annotated,
            self._adata_ids.assay_sc,
            self._adata_ids.assay_differentiation,
            self._adata_ids.assay_type_differentiation,
            self._adata_ids.author,
            self._adata_ids.cell_line,
            self._adata_ids.dataset,
            self._adata_ids.cell_type,
            self._adata_ids.development_stage,
            self._adata_ids.normalization,
            self._adata_ids.organ,
            self._adata_ids.bio_sample,
            self._adata_ids.state_exact,
            self._adata_ids.year,
        ]
        if scatter_update:
            self.adata.obs = pandas.DataFrame({
                k: ["nan" for _ in range(n_cells)] for k in keys
            })
        else:
            for k in keys:
                self.adata.obs[k] = []
        # Define index vectors to write to:
        idx_vector = np.arange(0, n_cells)
        if shuffled:
            np.random.shuffle(idx_vector)
        idx_ls = []
        row = 0
        ncells = self.ncells_bydataset(annotated_only=annotated_only)
        if np.all([len(x) == 0 for x in ncells]):
            raise ValueError("no datasets found")
        for x in ncells:
            temp_ls = []
            for y in x:
                temp_ls.append(idx_vector[row:(row + y)])
                row += y
            idx_ls.append(temp_ls)
        print("checking expected and received data set sizes, rerun meta data generation if mismatch is found:")
        print(self.ncells_bydataset(annotated_only=annotated_only))
        print([[len(x) for x in xx] for xx in idx_ls])
        for i, x in enumerate(self.dataset_groups):
            x.write_backed(
                adata_backed=self.adata,
                genome=genome,
                idx=idx_ls[i],
                annotated_only=annotated_only,
                load_raw=load_raw,
                allow_caching=allow_caching,
            )
        # If the sparse non-shuffled approach is used, make sure that self.adata.obs.index is unique() before saving
        if not scatter_update:
            self.adata.obs.index = pd.RangeIndex(0, len(self.adata.obs.index))
        # Explicitly write backed file to disk again to make sure that obs are included and that n_obs is set correctly
        self.adata.write()
        # Saving obs separately below is therefore no longer required (hence commented out)
        # fn_backed_obs = ".".join(self.fn_backed.split(".")[:-1]) + "_obs.csv"
        # self.adata.obs.to_csv(fn_backed_obs)

    def delete_backed(self):
        del self.adata
        self.adata = None
        os.remove(str(self.fn_backed))

    def load_cached_backed(self, fn: PathLike):
        self.adata = anndata.read(fn, backed='r')

    def streamline_metadata(
            self,
            schema: str = "sfaira",
            clean_obs: bool = True,
            clean_var: bool = True,
            clean_uns: bool = True,
            clean_obs_names: bool = True,
            keep_orginal_obs: bool = False,
            keep_symbol_obs: bool = True,
            keep_id_obs: bool = True,
    ):
        """
        Streamline the adata instance in each group and each data set to output format.
        Output format are saved in ADATA_FIELDS* classes.

        :param schema: Export format.
            - "sfaira"
            - "cellxgene"
        :param clean_obs: Whether to delete non-streamlined fields in .obs, .obsm and .obsp.
        :param clean_var: Whether to delete non-streamlined fields in .var, .varm and .varp.
        :param clean_uns: Whether to delete non-streamlined fields in .uns.
        :param clean_obs_names: Whether to replace obs_names with a string comprised of dataset id and an increasing integer.
        :param clean_obs_names: Whether to replace obs_names with a string comprised of dataset id and an increasing
            integer.
        :param keep_orginal_obs: For ontology-constrained .obs columns, whether to keep a column with original
            annotation.
        :param keep_symbol_obs: For ontology-constrained .obs columns, whether to keep a column with ontology symbol
            annotation.
        :param keep_id_obs: For ontology-constrained .obs columns, whether to keep a column with ontology ID annotation.
        :return:
        """
        for x in self.dataset_groups:
            for xx in x.ids:
                x.datasets[xx].streamline_metadata(
                    schema=schema,
                    clean_obs=clean_obs,
                    clean_var=clean_var,
                    clean_uns=clean_uns,
                    clean_obs_names=clean_obs_names,
                    keep_orginal_obs=keep_orginal_obs,
                    keep_symbol_obs=keep_symbol_obs,
                    keep_id_obs=keep_id_obs,
                )

    def subset(self, key, values):
        """
        Subset list of adata objects based on match to values in key property.

        These keys are properties that are available in lazy model.
        Subsetting happens on .datasets.

        :param key: Property to subset by.
        :param values: Classes to overlap to.
        :return:
        """
        for x in self.dataset_groups:
            x.subset(key=key, values=values)
        self.dataset_groups = [x for x in self.dataset_groups if x.datasets]  # Delete empty DatasetGroups

    def remove_duplicates(
            self,
            supplier_hierarchy: str = "cellxgene,sfaira"
    ):
        """
        Remove duplicate data loaders from super group, e.g. loaders that map to the same DOI.

        Any DOI match is removed (pre-print or journal publication).
        Data sets without DOI are removed, too.
        Loaders are kept in the hierarchy indicated in supplier_hierarchy.
        Requires a super group with homogenous suppliers across DatasetGroups, throws an error otherwise.
        This is given for sfaira maintained libraries but may not be the case if custom assembled DatasetGroups are
        used.

        :param supplier_hierarchy: Hierarchy to resolve duplications by.
            Comma separated string that indicates which data provider takes priority.
            Choose "cellxgene,sfaira" to prioritise use of data sets downloaded from cellxgene.
            Choose "sfaira,cellxgene" to prioritise use of raw data processing pipelines locally.

                - cellxgene: cellxgene downloads
                - sfaira: local raw file processing
        :return:
        """
        # Build a pairing of provider and DOI:
        report_list = []
        idx_tokeep = []
        supplier_hierarchy = supplier_hierarchy.split(",")
        for i, (x, y) in enumerate([(xx.supplier, xx.doi) for xx in self.dataset_groups]):
            if len(x) > 1:
                raise ValueError(f"found more than one supplier for DOI {str(y)}")
            else:
                x = x[0]
                if x not in supplier_hierarchy:
                    raise ValueError(f"could not associate supplier {x} with hierarchy {supplier_hierarchy} in "
                                     f"data set {y}")
                if len(report_list) > 0:
                    matched_idx = np.where([
                        np.any([
                            zz in y
                            for zz in z[1]
                        ])
                        for z in report_list
                    ])[0]
                    assert len(matched_idx) < 1, f"more matches than expected for {(x, y)}"
                else:
                    matched_idx = []
                if len(matched_idx) > 0:
                    # Establish which entry takes priority
                    supplier_old = report_list[matched_idx[0]][0]
                    priority = supplier_hierarchy.index(supplier_old) > supplier_hierarchy.index(x)
                    print(f"removing duplicate data set {y} from supplier: {supplier_old if priority else x}")
                    if priority:
                        idx_tokeep.append(i)
                        del idx_tokeep[matched_idx[0]]
                else:
                    report_list.append([x, y])
                    idx_tokeep.append(i)
        self.dataset_groups = [self.dataset_groups[i] for i in idx_tokeep]

    def subset_cells(self, key, values: Union[str, List[str]]):
        """
        Subset list of adata objects based on cell-wise properties.

        These keys are properties that are not available in lazy model and require loading first because the
        subsetting works on the cell-level: .adata are maintained but reduced to matches.

        :param key: Property to subset by. Options:

            - "assay_sc" points to self.assay_sc_obs_key
            - "assay_differentiation" points to self.assay_differentiation_obs_key
            - "assay_type_differentiation" points to self.assay_type_differentiation_obs_key
            - "cell_line" points to self.cell_line
            - "cell_type" points to self.cell_type_obs_key
            - "developmental_stage" points to self.developmental_stage_obs_key
            - "ethnicity" points to self.ethnicity_obs_key
            - "organ" points to self.organ_obs_key
            - "organism" points to self.organism_obs_key
            - "sample_source" points to self.sample_source_obs_key
            - "sex" points to self.sex_obs_key
            - "state_exact" points to self.state_exact_obs_key
        :param values: Classes to overlap to.
        :return:
        """
        for i in range(len(self.dataset_groups)):
            self.dataset_groups[i].subset_cells(key=key, values=values)

    def project_celltypes_to_ontology(self, adata_fields: Union[AdataIds, None] = None, copy=False):
        """
        Project free text cell type names to ontology based on mapping table.
        :return:
        """
        for _, v in self.dataset_groups:
            v.project_free_to_ontology(adata_fields=adata_fields, copy=copy)

    def write_config(self, fn: Union[str, os.PathLike]):
        """
        Writes a config file that describes the current data sub-setting.

        This config file can be loaded later to recreate a sub-setting.

        :param fn: Output file.
        """
        pd.DataFrame({"id": np.sort(self.ids)}).to_csv(fn, index=False, sep="\t")

    def load_config(self, fn: Union[str, os.PathLike]):
        """
        Load a config file and recreates a data sub-setting.

        :param fn: Output file.
        """
        tab = pd.read_csv(fn, header=0, index_col=None, sep="\t")
        ids_keep = tab["id"].values
        self.subset(key="id", values=ids_keep)

    @property
    def additional_annotation_key(self) -> List[Dict[str, Union[None, str]]]:
        """"
        Return list (by data set group) of dictionaries of additional_annotation_key for each data set with ids as keys.
        """
        return [
            dict([
                (k, x.datasets[k].additional_annotation_key)
                for k, v in x.datasets.items()
            ]) for x in self.dataset_groups
        ]

    @additional_annotation_key.setter
    def additional_annotation_key(self, x: Dict[str, Union[None, str]]):
        """
        Allows setting of additional_annotation_key in a subset of datasets identifed by keys in x.

        The input is not structured by DatasetGroups but only by ID, all groups are checked for matching IDs.

        :param x: Dictionary with data set ids in keys and new _additional_annotation_key values to be setted in values.
            Note that you can either add  or change secondary annotation by setting a value to a string or remove it
            by setting a value to None.
        :return:
        """
        for k, v in x.items():
            counter = 0
            for x in self.dataset_groups:
                if k in x.ids:
                    x.datasets[k].additional_annotation_key = v
                    counter += 1
            if counter == 0:
                warnings.warn(f"did not data set matching ID {k}")
            elif counter > 1:
                warnings.warn(f"found more than one ({counter}) data set matching ID {k}")

    def show_summary(self):
        for k, v in self.datasets.items():
            print(k)
            print(f"\t {(v.supplier, v.organism, v.organ, v.assay_sc, v.disease)}")


from lib.list import Node, create_palindrome_list

def is_palindrome(head):
    base = head
    runner = head
    stack = []

    while runner is not None and runner.next is not None:
        stack.append(base.data)
        base = base.next
        runner = runner.next.next

    if runner is not None:
        base = base.next

    while base is not None:
        if stack.pop() != base.data:
            return False
        base = base.next

    return True

if __name__ == '__main__':
    head = create_palindrome_list(7)
    print(f'Palindrome: {head}')
    print(is_palindrome(head))


_author_ = 'Mkoijn'

'''Encrypt given plaintext and Decrypt given cyphertext'''

random_text = input("Enter plaintext: ")
print('\nEncrypting plaintext...')

new_text = random_text[:: -1]
crypt_text = ''
for character in new_text:
    x = ord(character)
    crypt_text += chr(x + 1)
    print(character, "->", (chr(x + 1)))

print('\nEncrypted string is...',crypt_text)

cypher_text = input("\nEnter cyphertext: ")
print('\nDecrypting cyphertext...')

rev_cypher_text = cypher_text[::-1]
plain_text = ''
for character in rev_cypher_text:
    x = ord(character)
    plain_text += chr(x - 1)
    print(character, "->", (chr(x - 1)))

print('\nDecrypted string is...',plain_text)


# http://qiita.com/tkazusa/items/b8e4d287f94f8b87ed52

import matplotlib.pyplot as plt
from pylab import *
import numpy as np
import random

def heaviside(x):
    return 0.5 * (np.sign(x) + 1)

def NN(x_train, t, n_imput, n_hidden, n_output, eta, W1, W2, n_loop):
    fp_w1 = open("W1.txt", "w")
    fp_w2 = open("W2.txt", "w")
    fp_diff1 = open("diff1.txt", "w")
    fp_diff2 = open("diff2.txt", "w")

    for n in range(n_loop):
        header = "# loop %d\n" % n
        fp_w1.write(header)
        fp_w2.write(header)
        fp_diff1.write(header)
        fp_diff2.write(header)

        for n in range(len(x_train)):
            x = np.array([x_train[n]])

            #feedforward
            X = np.insert(x, 0, 1) #Insert fixed term

            A = np.dot(W1, X) #(5.62)
            Z = np.tanh(A)  #(5.63)
            Z[0] = 1.0
            Y = np.dot(W2, Z) #(5.64)

            #Backprobagation
            D2 = Y - t[n]#(5.65)
            D1 = (1-Z**2)*W2*D2 #(5.66)

            diff_w1 = W1 - eta*D1.T*X
            diff_w2 = W2 - eta*D2.T*Z

            W1 = diff_w1 #(5.67), (5.43)
            W2 = diff_w2 #(5.67), (5.43)

            str_w1 = "%s\n" % str(W1)
            str_w2 = "%s\n" % str(W2)
            str_diff1 = "%s\n" % str(diff_w1)
            str_diff2 = "%s\n" % str(diff_w2)
            fp_w1.write(str_w1)
            fp_w2.write(str_w2)
            fp_diff1.write(str_diff1)
            fp_diff2.write(str_diff2)

    fp_w1.close()
    fp_w2.close()
    fp_diff1.close()
    fp_diff2.close()

    return  W1, W2

def output(x, W1, W2):
    X = np.insert(x, 0, 1) #Insert fixed term

    A = np.dot(W1, X) #(5.62)
    Z = np.tanh(A)  #(5.63)
    Z[0] = 1.0 #Insert fixed term
    Y = np.dot(W2, Z) #(5.64)
    return Y, Z

def print_weight(w, name):
    fp = open(name+".txt", "w")
    s = str(w)
    fp.write(s)
    fp.close()

if __name__ == "__main__":
#    global fp_w1, fp_w2
    #Set form of nueral network
    n_imput = 2
    n_hidden = 4
    n_output = 1
    eta = 0.1
    W1 = np.random.random((n_hidden, n_imput))
    W2 = np.random.random((n_output, n_hidden))
    print_weight(W1, "W1_init")
    print_weight(W2, "W2_init")

    n_loop = 1000

    #Set train data
    x_train = np.linspace(-4, 4, 300).reshape(300, 1)
    y_train_1 = x_train * x_train
    y_train_2 = np.sin(x_train)
    y_train_3 = np.abs(x_train)
    y_train_4 = heaviside(x_train)

    fp_xtrain = open("xtrain.dat", "w")
    fp_xtrain_1 = open("xtrain_1.dat", "w")
    for x in x_train:
        fp_xtrain.write("%f\n" % x)
    for x in y_train_1:
        fp_xtrain_1.write("%f\n" % x)
    fp_xtrain.close()
    fp_xtrain_1.close()

    W1_1, W2_1= NN(x_train, y_train_1, n_imput, n_hidden, n_output, eta, W1, W2, n_loop)
    # W1_2, W2_2= NN(x_train, y_train_2, n_imput, n_hidden, n_output, eta, W1, W2, n_loop)
    # W1_3, W2_3= NN(x_train, y_train_3, n_imput, n_hidden, n_output, eta, W1, W2, n_loop)
    # W1_4, W2_4= NN(x_train, y_train_4, n_imput, n_hidden, n_output, eta, W1, W2, n_loop)
    print_weight(W1_1, "W1_1_last")
    print_weight(W2_1, "W2_2_last")

    Y_1 = np.zeros((len(x_train), n_output))
    Z_1 = np.zeros((len(x_train), n_hidden))

#    Y_2 = np.zeros((len(x_train), n_output))
#    Z_2 = np.zeros((len(x_train), n_hidden))

#    Y_3 = np.zeros((len(x_train), n_output))
#    Z_3 = np.zeros((len(x_train), n_hidden))

#    Y_4 = np.zeros((len(x_train), n_output))
#    Z_4 = np.zeros((len(x_train), n_hidden))

    for n in range(len(x_train)):
        Y_1[n], Z_1[n] =output(x_train[n], W1_1, W2_1)
       # Y_2[n], Z_2[n] =output(x_train[n], W1_2, W2_2)
       # Y_3[n], Z_3[n] =output(x_train[n], W1_3, W2_3)
       # Y_4[n], Z_4[n] =output(x_train[n], W1_4, W2_4)

    plt.plot(x_train, Y_1, "r-")
    plt.plot(x_train, y_train_1, "bo", markersize=3)
    for i in range(n_hidden):
        plt.plot(x_train, Z_1[:,i], 'm--')
    xlim([-1,1])
    ylim([0, 1])
    title("Figure 5.3(a)")
    show()

    # plt.plot(x_train, Y_2, "r-")
    # plt.plot(x_train, y_train_2, "bo", markersize=2)
    # for i in range(n_hidden):
    #     plt.plot(x_train, Z_2[:,i], 'm--')
    # xlim([-3.14,3.14])
    # ylim([-1, 1])
    # title("Figure 5.3(b)")
    # show()
    #
    #
    # plt.plot(x_train, Y_3, "r-")
    # plt.plot(x_train, y_train_3, "bo", markersize=4)
    # for i in range(n_hidden):
    #     plt.plot(x_train, Z_3[:,i], 'm--')
    # xlim([-1,1])
    # ylim([0, 1])
    # title("Figure 5.3(c)")
    # show()
    #
    #
    # plt.plot(x_train, Y_4, "r-")
    # plt.plot(x_train, y_train_4, "bo" ,markersize=2)
    # for i in range(n_hidden):
    #     plt.plot(x_train, Z_4[:,i], 'm--')
    # xlim([-2,2])
    # ylim([-0.05, 1.05])
    # title("Figure 5.3(d)")
    # show()