text:
#ifndef EV3PLOTTER_MESSAGEQUEUE_H
#define EV3PLOTTER_MESSAGEQUEUE_H

#include <gsl/span>
#include <memory>
#include <string_view>
#include <type_safe/flag_set.hpp>

namespace ev3plotter {

class message_queue {
  public:
    enum class option { read, write, non_blocking, remove_on_destruction, _flag_set_size };

    enum class send_result { success, failure_queue_full, failure };

    enum class receive_result { success, failure_no_messages, failure };

    message_queue(std::string_view name, std::size_t max_message_size, type_safe::flag_set<option> options);
    ~message_queue();

    send_result send(std::string_view msg);
    receive_result receive(gsl::span<char>& buffer);

    std::size_t message_size() const noexcept;

  private:
    class impl;

    std::unique_ptr<impl> impl_;
};
} // namespace ev3plotter

#endif // EV3PLOTTER_MESSAGEQUEUE_H

ext: h


text:
#include <gsl/gsl_test.h>
#include <gsl/gsl_ieee_utils.h>
#include <gsl/gsl_math.h>
#include "gsl_cblas.h"

#include "tests.h"

void
test_nrm2 (void) {
const double flteps = 1e-4, dbleps = 1e-6;
  {
   int N = 1;
   float X[] = { 0.317f };
   int incX = -1;
   float expected = 0.0f;
   float f;
   f = cblas_snrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "snrm2(case 28)");
  };

  {
   int N = 1;
   double X[] = { 0.071 };
   int incX = -1;
   double expected = 0;
   double f;
   f = cblas_dnrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dnrm2(case 29)");
  };

  {
   int N = 1;
   float X[] = { 0.776f, 0.983f };
   int incX = -1;
   float expected = 0.0f;
   float f;
   f = cblas_scnrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "scnrm2(case 30)");
  };

  {
   int N = 1;
   double X[] = { 0.549, -0.354 };
   int incX = -1;
   double expected = 0;
   double f;
   f = cblas_dznrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dznrm2(case 31)");
  };

  {
   int N = 2;
   float X[] = { 0.14f, -0.632f };
   int incX = 1;
   float expected = 0.647320631527f;
   float f;
   f = cblas_snrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "snrm2(case 32)");
  };

  {
   int N = 2;
   double X[] = { 0.696, -0.804 };
   int incX = 1;
   double expected = 1.06340584915;
   double f;
   f = cblas_dnrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dnrm2(case 33)");
  };

  {
   int N = 2;
   float X[] = { 0.281f, -0.063f, 0.367f, 0.232f };
   int incX = 1;
   float expected = 0.521001919382f;
   float f;
   f = cblas_scnrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "scnrm2(case 34)");
  };

  {
   int N = 2;
   double X[] = { -0.359, -0.76, -0.906, -0.108 };
   int incX = 1;
   double expected = 1.24055672986;
   double f;
   f = cblas_dznrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dznrm2(case 35)");
  };

  {
   int N = 2;
   float X[] = { 0.918f, -0.126f };
   int incX = -1;
   float expected = 0.0f;
   float f;
   f = cblas_snrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "snrm2(case 36)");
  };

  {
   int N = 2;
   double X[] = { 0.217, -0.588 };
   int incX = -1;
   double expected = 0;
   double f;
   f = cblas_dnrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dnrm2(case 37)");
  };

  {
   int N = 2;
   float X[] = { 0.31f, 0.059f, -0.442f, 0.987f };
   int incX = -1;
   float expected = 0.0f;
   float f;
   f = cblas_scnrm2(N, X, incX);
   gsl_test_rel(f, expected, flteps, "scnrm2(case 38)");
  };

  {
   int N = 2;
   double X[] = { 0.609, 0.615, -0.143, -0.957 };
   int incX = -1;
   double expected = 0;
   double f;
   f = cblas_dznrm2(N, X, incX);
   gsl_test_rel(f, expected, dbleps, "dznrm2(case 39)");
  };

}

ext: c


text:
#if MKL_FOUND
#include <mkl.h>
#else
#if BLAS_FOUND
#include <cblas.h>
#endif
#endif

inline void sgemm(bool transA,
                  bool transB,
                  int rows_a,
                  int rows_b,
                  int width,
                  float alpha,
                  float* a,
                  int lda,
                  float* b,
                  int ldb,
                  float beta,
                  float* c,
                  int ldc) {
#if BLAS_FOUND
  cblas_sgemm(CblasRowMajor,
              transA ? CblasTrans : CblasNoTrans,
              transB ? CblasTrans : CblasNoTrans,
              rows_a,
              rows_b,
              width,
              alpha,
              a,
              lda,
              b,
              ldb,
              beta,
              c,
              ldc);
#else
    transA; transB; rows_a; rows_b; width; alpha; a; lda; b; ldb; beta; c; ldc; // make compiler happy
    ABORT("Marian must be compiled with a BLAS library");
#endif
}

max_stars_repo_name: delong-coder/marian-dev

max_issues_repo_name: delong-coder/marian-dev

max_forks_repo_name: delong-coder/marian-dev


text:
#include<stdio.h>
#include<sys/time.h>
#include <stdlib.h>
#include <cblas.h>
#include <omp.h>

void SGEMM_NT_MP(float *C, float *A, float *B, long	M, long N, long K,
			long LN, long LK, float *SB, long k_tag)
{
	asm volatile(
		".macro PACK_KERNEL5x4_BEGIN_K             	 \n"
		"										 	 \n"

		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16				     \n"
		"	ldr		q5, [x16], #16					 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"
		"	prfm	PLDL1KEEP, [x15, #64]			 \n"

		"	prfm 	PLDL1KEEP, [x17, #128]			 \n"

		"	fmul	v12.4s, v0.4s, v5.4s			 \n"
		"	ldr		q2, [x13], #16					 \n"
		"	fmul	v13.4s, v1.4s, v5.4s			 \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmul	v14.4s, v2.4s, v5.4s		 	 \n"
		"	ldr		q4, [x15], #16					 \n"
		"	fmul	v15.4s, v3.4s, v5.4s			 \n"

		"	ldr		q6, [x17], #16					 \n"
		"	fmul	v16.4s, v4.4s, v5.4s			 \n"
		"	ldr		q7, [x18], #16					 \n"
		"	fmul	v17.4s, v0.4s, v6.4s			 \n"

		"	ldr		q8, [x19], #16					 \n"
		"	ldr		q9, [x11], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[0], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmul	v18.4s, v1.4s, v6.4s			 \n"
		"	fmul	v19.4s, v2.4s, v6.4s			 \n"
		"	fmul	v20.4s, v3.4s, v6.4s			 \n"
		"	fmul	v21.4s, v4.4s, v6.4s			 \n"

		"	ldr		q10, [x12], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[1], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmul	v22.4s, v0.4s, v7.4s			 \n"
		"	fmul	v23.4s, v1.4s, v7.4s			 \n"
		"	fmul	v24.4s, v2.4s, v7.4s			 \n"
		"	fmul	v25.4s, v3.4s, v7.4s			 \n"
		"	fmul	v26.4s, v4.4s, v7.4s			 \n"

		"	ldr		q11, [x13], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[2], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmul	v27.4s, v0.4s, v8.4s			 \n"
		"	ldr		q0, [x14], #16					 \n"
		"	fmul	v28.4s, v1.4s, v8.4s			 \n"
		"	ldr		q1, [x15], #16					 \n"

		"	st4		{v5.s, v6.s, v7.s, v8.s}[3], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"
		"	ldr		q5, [x16], #16					 \n"

		"	fmul	v29.4s, v2.4s, v8.4s			 \n"
		"	fmul	v30.4s, v3.4s, v8.4s			 \n"
		"	fmul	v31.4s, v4.4s, v8.4s			 \n"

		".endm									 	 \n"

		".macro PACK_KERNEL5x4_K0                    \n"

		"	prfm 	PLDL1KEEP, [x16, #128]			 \n"

		"	ldr		q6, [x17], #16					 \n"

		"	fmla	v12.4s, v0.4s, v5.4s			 \n"
		"	fmla	v13.4s, v1.4s, v5.4s			 \n"

		"	prfm 	PLDL1KEEP, [x17, #128]			 \n"

		"	ldr		q7, [x18], #16					 \n"

		"	prfm 	PLDL1KEEP, [x11, #64]			 \n"

		"	fmla	v14.4s, v2.4s, v5.4s			 \n"
		"	fmla	v15.4s, v3.4s, v5.4s			 \n"
		"	fmla	v16.4s, v4.4s, v5.4s			 \n"

		"	prfm 	PLDL1KEEP, [x12, #64]			 \n"

		"	ldr		q8, [x19], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[0], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v17.4s, v0.4s, v6.4s			 \n"
		"	fmla	v18.4s, v1.4s, v6.4s		 	 \n"
		"	fmla	v19.4s, v2.4s, v6.4s			 \n"
		"	fmla	v20.4s, v3.4s, v6.4s			 \n"
		"	fmla	v21.4s, v4.4s, v6.4s			 \n"

		"	ldr		q9, [x11], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[1], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v22.4s, v0.4s, v7.4s			 \n"
		"	fmla	v23.4s, v1.4s, v7.4s		 	 \n"

		"	ldr		q10, [x12], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[2], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v24.4s, v2.4s, v7.4s			 \n"
		"	fmla	v25.4s, v3.4s, v7.4s			 \n"
		"	fmla	v26.4s, v4.4s, v7.4s			 \n"

		"	ldr		q11, [x13], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[3], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v27.4s, v0.4s, v8.4s			 \n"
		"	ldr		q5, [x16], #16					 \n"
		"	ldr		q0, [x14], #16					 \n"
		"	fmla	v28.4s, v1.4s, v8.4s		 	 \n"
		"	ldr		q1, [x15], #16					 \n"

		"	fmla	v29.4s, v2.4s, v8.4s			 \n"
		"	fmla	v30.4s, v3.4s, v8.4s			 \n"
		"	fmla	v31.4s, v4.4s, v8.4s			 \n"

		".endm   									 \n"

		".macro PACK_KERNEL5x4_K1                    \n"

		"	prfm 	PLDL1KEEP, [x18, #128]			 \n"

		"	ldr		q6, [x17], #16					 \n"
		"	fmla	v12.4s, v9.4s, v5.4s 			 \n"
		"	fmla	v13.4s, v10.4s, v5.4s		  	 \n"

		"	prfm  	PLDL1KEEP, [x19, #128]			 \n"
		"	ldr		q7, [x18], #16					 \n"
		"	fmla	v14.4s, v11.4s, v5.4s		 	 \n"
		"	fmla	v15.4s, v0.4s, v5.4s			 \n"
		"	fmla	v16.4s, v1.4s, v5.4s			 \n"

		"	prfm 	PLDL1KEEP, [x13, #64]			 \n"
		"	ldr		q8, [x19], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[0], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v17.4s, v9.4s, v6.4s 			 \n"
		"	fmla	v18.4s, v10.4s, v6.4s		  	 \n"

		"	ldr		q2, [x13], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[1], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	prfm  	PLDL1KEEP, [x14, #64]			 \n"
		"	fmla	v19.4s, v11.4s, v6.4s		 	 \n"
		"	fmla	v20.4s, v0.4s, v6.4s			 \n"
		"	fmla	v21.4s, v1.4s, v6.4s			 \n"

		"	prfm  	PLDL1KEEP, [x15, #64]			 \n"
		"	ldr		q3, [x14], #16					 \n"

		"	fmla	v25.4s, v0.4s, v7.4s			 \n"
		"	fmla	v26.4s, v1.4s, v7.4s			 \n"

		"	st4		{v5.s, v6.s, v7.s, v8.s}[2], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v30.4s, v0.4s, v8.4s			 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	fmla	v31.4s, v1.4s, v8.4s			 \n"
		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v22.4s, v9.4s, v7.4s		  	 \n"

		"	st4		{v5.s, v6.s, v7.s, v8.s}[3], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"
		"	ldr		q5, [x16], #16					 \n"

		"	fmla	v23.4s, v10.4s, v7.4s			 \n"
		"	fmla	v24.4s, v11.4s, v7.4s			 \n"

		"	ldr		q4, [x15], #16					 \n"

		"	fmla	v27.4s, v9.4s, v8.4s			 \n"
		"	fmla	v28.4s, v10.4s, v8.4s			 \n"
		"	fmla	v29.4s, v11.4s, v8.4s			 \n"

		".endm									 	 \n"

		".macro PACK_KERNEL5x4_END_K                 \n"

		"	ldr		q6, [x17], #16					 \n"
		"	fmla	v12.4s, v9.4s, v5.4s 			 \n"
		"	fmla	v13.4s, v10.4s, v5.4s		  	 \n"

		"	ldr		q7, [x18], #16					 \n"
		"	fmla	v14.4s, v11.4s, v5.4s		 	 \n"
		"	fmla	v15.4s, v0.4s, v5.4s			 \n"
		"	fmla	v16.4s, v1.4s, v5.4s			 \n"

		"	ldr		q8, [x19], #16					 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[0], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v17.4s, v9.4s, v6.4s 			 \n"
		"	faddp	v12.4s, v12.4s, v17.4s			 \n"
		"	fmla	v18.4s, v10.4s, v6.4s		  	 \n"
		"	faddp	v13.4s, v13.4s, v18.4s			 \n"

		"	st4		{v5.s, v6.s, v7.s, v8.s}[1], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v19.4s, v11.4s, v6.4s		 	 \n"
		"	faddp	v14.4s, v14.4s, v19.4s			 \n"
		"	fmla	v20.4s, v0.4s, v6.4s			 \n"
		"	faddp	v15.4s, v15.4s, v20.4s			 \n"
		"	fmla	v21.4s, v1.4s, v6.4s			 \n"
		"	faddp	v16.4s, v16.4s, v21.4s			 \n"

		"	fmla	v25.4s, v0.4s, v7.4s			 \n"
		"	fmla	v26.4s, v1.4s, v7.4s			 \n"

		"	st4		{v5.s, v6.s, v7.s, v8.s}[2], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"

		"	fmla	v30.4s, v0.4s, v8.4s			 \n"
		"	faddp	v25.4s, v25.4s, v30.4s			 \n"
		"	fmla	v31.4s, v1.4s, v8.4s			 \n"
		"	faddp	v26.4s, v26.4s, v31.4s			 \n"

		"	fmla	v22.4s, v9.4s, v7.4s		  	 \n"
		"	st4		{v5.s, v6.s, v7.s, v8.s}[3], [x24]		\n"
		"	add		x24, x24, x8, lsl #2			 \n"
		"	fmla	v23.4s, v10.4s, v7.4s			 \n"
		"	fmla	v24.4s, v11.4s, v7.4s			 \n"

		"	faddp	v15.4s, v15.4s, v25.4s			 \n"
		"	faddp 	v16.4s, v16.4s, v26.4s			 \n"

		"	fmla	v27.4s, v9.4s, v8.4s			 \n"
		"	faddp 	v22.4s, v22.4s, v27.4s			 \n"
		"	fmla	v28.4s, v10.4s, v8.4s			 \n"
		"	faddp	v23.4s, v23.4s, v28.4s			 \n"
		"	fmla	v29.4s, v11.4s, v8.4s			 \n"
		"	faddp	v24.4s, v24.4s, v29.4s			 \n"

		"	faddp	v12.4s, v12.4s, v22.4s			 \n"
		"	faddp	v13.4s, v13.4s, v23.4s		 	 \n"
		"	faddp	v14.4s, v14.4s, v24.4s			 \n"

		".endm 										 \n"

		".macro	PACKB_ADD_C 						 \n"

		"	ldr		q0, [x25]						 \n"
		"	ldr		q1, [x26]						 \n"

		"	fadd 	v12.4s, v12.4s, v0.4s			 \n"
		"	ldr		q2, [x27]						 \n"
		"	fadd 	v13.4s, v13.4s, v1.4s			 \n"
		"	ldr		q3, [x28]						 \n"
		"	fadd 	v14.4s, v14.4s, v2.4s			 \n"
		"	ldr		q4, [x29]						 \n"
		"	fadd 	v15.4s, v15.4s, v3.4s			 \n"
		"	fadd 	v16.4s, v16.4s, v4.4s			 \n"

		".endm 									 	 \n"

		".macro SAVE5x4							 	 \n"

		"	subs	x7, x7, #1					 	 \n"

		"	str		q12, [x25], #16					 \n"
		"	str		q13, [x26], #16					 \n"
		"	str		q14, [x27], #16		 			 \n"
		"	str		q15, [x28], #16		 			 \n"
		"	str		q16, [x29], #16					 \n"

		".endm 									 	 \n"

		".macro ADD_C								 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"
		"	prfm	PLDL1KEEP, [x29, #64]			 \n"

		"	ldp		q0, q1, [x25]					 \n"

		"	fadd	v12.4s, v12.4s, v0.4s			 \n"
		"	ldp		q2, q3, [x25, #32]				 \n"
		"	fadd	v13.4s, v13.4s, v1.4s			 \n"

		"	fadd	v22.4s, v2.4s, v22.4s			 \n"
		"	ldp		q4, q5, [x26]					 \n"
		"	fadd	v23.4s, v3.4s, v23.4s		 	 \n"

		"	fadd	v14.4s, v4.4s, v14.4s			 \n"
		"	ldp		q6, q7, [x26, #32]				 \n"
		"	fadd	v15.4s, v5.4s, v15.4s			 \n"

		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"
		"	prfm	PLDL1KEEP, [x29, #64]			 \n"

		"	fadd	v24.4s, v6.4s, v24.4s			 \n"
		"	ldp		q8, q9, [x27]					 \n"
		"	fadd 	v25.4s, v7.4s, v25.4s			 \n"

		"	fadd	v16.4s, v8.4s, v16.4s			 \n"
		"	ldp		q10, q11, [x27, #32]			 \n"
		"	fadd	v17.4s, v9.4s, v17.4s			 \n"

		"	fadd	v26.4s, v10.4s, v26.4s			 \n"
		"	ldp		q0, q1, [x28]				  	 \n"
		"	fadd	v27.4s, v11.4s, v27.4s			 \n"

		"	fadd	v18.4s, v0.4s, v18.4s			 \n"
		"	ldp		q2, q3, [x28, #32]				 \n"
		"	fadd	v19.4s, v1.4s, v19.4s			 \n"

		"	fadd	v28.4s, v2.4s, v28.4s			 \n"
		"	ldp		q4, q5, [x29]					 \n"
		"	fadd	v29.4s, v3.4s, v29.4s			 \n"

		"	fadd	v20.4s, v4.4s, v20.4s			 \n"
		"	ldp		q6, q7, [x29, #32]				 \n"
		"	fadd	v21.4s, v5.4s, v21.4s			 \n"

		"	fadd	v30.4s, v30.4s, v6.4s			 \n"
		"	fadd	v31.4s, v31.4s, v7.4s		     \n"

		".endm 										 \n"

		".macro SAVE5x16							 \n"

		"	subs	x21, x21, #1					 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	stp		q16, q17, [x27]					 \n"
		"	stp		q18, q19, [x28]					 \n"
		"	stp		q20, q21, [x29]					 \n"

//		"	add 	x10, x10, x9					 \n"

		"	add 	x10, x10, x6, lsl #4			 \n"
		"	add		x10, x10, x6, lsl #2 			 \n"

		"	stp		q22, q23, [x25, #32]			 \n"
		"	add		x25, x29, x9, lsl #2					 \n"
		"	stp		q24, q25, [x26, #32]			 \n"
		"	add		x26, x29, x9, lsl #3			 \n"
		"	stp		q26, q27, [x27, #32]			 \n"
		"	add		x27, x25, x9, lsl #3 			 \n"
		"	stp		q28, q29, [x28, #32]			 \n"
		"	add		x28, x26, x9, lsl #3 			 \n"
		"	stp		q30, q31, [x29, #32]			 \n"
		"	add		x29, x27, x9, lsl #3			 \n"

		".endm 									 	 \n"

		".macro KERNEL5x16_BEGIN_K             	 	 \n"
		"										 	 \n"

		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"
		"	prfm	PLDL1KEEP, [x15, #64]			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"

		"	ldr		q4, [x15], #16				 	 \n"

		"	fmul	v18.4s, v8.4s, v3.s[0]	         \n"
		"	fmul	v19.4s, v9.4s, v3.s[0]	         \n"

		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v20.4s, v8.4s, v4.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmul	v21.4s, v9.4s, v4.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmul	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmul	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmul	v24.4s, v10.4s, v1.s[0]	         \n"
		"	fmul	v25.4s, v11.4s, v1.s[0]	         \n"

		"	fmul	v26.4s, v10.4s, v2.s[0]	         \n"
		"	fmul	v27.4s, v11.4s, v2.s[0]	         \n"

		"	ldr		q5, [x11], #16					 \n"

		"	fmul	v28.4s, v10.4s, v3.s[0]	         \n"
		"	fmul	v29.4s, v11.4s, v3.s[0]	         \n"

		"	fmul	v30.4s, v10.4s, v4.s[0]	         \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmul	v31.4s, v11.4s, v4.s[0]	         \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL5x16_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[0]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[0]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v4.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[0]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[0]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[0]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[0]			 \n"

		"	ldr		q5, [x11], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v3.s[0]			 \n"
		"	fmla	v29.4s, v11.4s, v3.s[0]			 \n"

		"	fmla	v30.4s, v10.4s, v4.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v4.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm   									 \n"

		".macro KERNEL5x16_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[1]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[1]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[1]			 \n"
		"	fmla	v19.4s, v9.4s, v3.s[1]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v4.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[1]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[1]			 \n"

		"	ldr		q6, [x12], #16					 \n"

		"	fmla	v28.4s, v10.4s, v3.s[1]			 \n"
		"	fmla	v29.4s, v11.4s, v3.s[1]			 \n"

		"	fmla	v30.4s, v10.4s, v4.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v4.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL5x16_K2          		         \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[2]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[2]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v4.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[2]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[2]			 \n"

		"	prfm	PLDL1KEEP, [x14, #64]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[2]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[2]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[2]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[2]			 \n"

		"	ldr		q7, [x13], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v3.s[2]			 \n"
		"	fmla	v29.4s, v11.4s, v3.s[2]			 \n"

		"	fmla	v30.4s, v10.4s, v4.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v4.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL5x16_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x15, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[3]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v9.4s, v3.s[3]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[3]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v4.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x14], #16					 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x15], #16					 \n"

		"	fmla	v26.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[3]			 \n"

		"	fmla	v28.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v29.4s, v11.4s, v3.s[3]			 \n"

		"	fmla	v30.4s, v10.4s, v4.s[3]			 \n"
		"	ldr		q10, [x24], #16				 	 \n"
		"	fmla	v31.4s, v11.4s, v4.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL5x16_K4    	                 \n"

		"	prfm	PLDL1KEEP, [x13, #64]			 \n"

		"	fmla	v12.4s, v8.4s, v5.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v5.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v6.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v6.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v7.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v7.s[0]			 \n"

		"	fmla	v18.4s, v8.4s, v0.s[0]			 \n"
		"	fmla 	v19.4s, v9.4s, v0.s[0]			 \n"

		"	fmla	v20.4s, v8.4s, v1.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v1.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v5.s[0]			 \n"
		"	fmla	v23.4s, v11.4s, v5.s[0]			 \n"

		"	fmla	v24.4s, v10.4s, v6.s[0]			 \n"
		"	fmla	v25.4s, v11.4s, v6.s[0]			 \n"

		"	fmla	v26.4s, v10.4s, v7.s[0]			 \n"
		"	fmla	v27.4s, v11.4s, v7.s[0]			 \n"

		"	ldr		q2, [x13], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v0.s[0]			 \n"
		"	fmla	v29.4s, v11.4s, v0.s[0]			 \n"

		"	fmla	v30.4s, v10.4s, v1.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v1.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL5x16_K5         	             \n"

		"   prfm    PLDL1KEEP, [x14, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v5.s[1]			 \n"
		"	fmla	v13.4s, v9.4s, v5.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v6.s[1]			 \n"
		"	fmla	v15.4s, v9.4s, v6.s[1]			 \n"

		"	fmla	v16.4s, v8.4s, v7.s[1]			 \n"
		"	fmla	v17.4s, v9.4s, v7.s[1]			 \n"

		"	fmla	v18.4s, v8.4s, v0.s[1]			 \n"
		"	fmla	v19.4s, v9.4s, v0.s[1]			 \n"

		"	fmla	v20.4s, v8.4s, v1.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v1.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v5.s[1]			 \n"
		"	fmla	v23.4s, v11.4s, v5.s[1]			 \n"

		"	fmla	v24.4s, v10.4s, v6.s[1]			 \n"
		"	fmla	v25.4s, v11.4s, v6.s[1]			 \n"

		"	fmla	v26.4s, v10.4s, v7.s[1]			 \n"
		"	fmla	v27.4s, v11.4s, v7.s[1]			 \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmla	v28.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v29.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v30.4s, v10.4s, v1.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v1.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL5x16_K6 	                     \n"

		"	prfm	PLDL1KEEP, [x15, #64]			 \n"

		"	fmla	v12.4s, v8.4s, v5.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v5.s[2]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v6.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v6.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v7.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v7.s[2]			 \n"

		"	fmla	v18.4s, v8.4s, v0.s[2]			 \n"
		"	fmla 	v19.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v20.4s, v8.4s, v1.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v21.4s, v9.4s, v1.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v5.s[2]			 \n"
		"	fmla	v23.4s, v11.4s, v5.s[2]			 \n"

		"	fmla	v24.4s, v10.4s, v6.s[2]			 \n"
		"	fmla	v25.4s, v11.4s, v6.s[2]			 \n"

		"	prfm	PLDL1KEEP, [x11, #64]			 \n"

		"	fmla	v26.4s, v10.4s, v7.s[2]			 \n"
		"	fmla	v27.4s, v11.4s, v7.s[2]			 \n"

		"	ldr		q4, [x15], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v0.s[2]			 \n"
		"	fmla	v29.4s, v11.4s, v0.s[2]			 \n"

		"	fmla	v30.4s, v10.4s, v1.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v31.4s, v11.4s, v1.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL5x16_K7      		             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v5.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v5.s[3]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v6.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v6.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v7.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v7.s[3]			 \n"

		"	fmla	v18.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v19.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v20.4s, v8.4s, v1.s[3]			 \n"
		"	ldr		q8, [x24], #16			     	 \n"
		"	fmla	v21.4s, v9.4s, v1.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v28.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v29.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		"	fmla	v30.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v31.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v22.4s, v10.4s, v5.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v5.s[3]			 \n"

		"	fmla	v24.4s, v10.4s, v6.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v6.s[3]			 \n"

		"	fmla	v26.4s, v10.4s, v7.s[3]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v27.4s, v11.4s, v7.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL5x16_END_K                	 \n"

		"	fmla	v12.4s, v8.4s, v5.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v5.s[3]			 \n"

		"	fmla	v14.4s, v8.4s, v6.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v6.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v7.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v7.s[3]			 \n"

		"	fmla	v18.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v19.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v20.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v21.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v28.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v29.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v30.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v31.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v22.4s, v10.4s, v5.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v5.s[3]			 \n"

		"	fmla	v24.4s, v10.4s, v6.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v6.s[3]			 \n"

		"	fmla	v26.4s, v10.4s, v7.s[3]			 \n"
		"	fmla	v27.4s, v11.4s, v7.s[3]			 \n"

		".endm 										 \n"

		".macro	KERNEL4x16_BEGIN_K					 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v18.4s, v8.4s, v3.s[0]	         \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmul	v19.4s, v9.4s, v3.s[0]	         \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmul	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmul	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmul	v24.4s, v10.4s, v1.s[0]	         \n"
		"	fmul	v25.4s, v11.4s, v1.s[0]	         \n"

		"	fmul	v26.4s, v10.4s, v2.s[0]	         \n"
		"	fmul	v27.4s, v11.4s, v2.s[0]	         \n"

//		"	ldr		q5, [x11], #16					 \n"

		"	fmul	v28.4s, v10.4s, v3.s[0]	         \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmul	v29.4s, v11.4s, v3.s[0]	         \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL4x16_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[0]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[0]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[0]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[0]			 \n"

//		"	ldr		q5, [x11], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v3.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v29.4s, v11.4s, v3.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm   									 \n"

		".macro KERNEL4x16_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[1]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[1]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v19.4s, v9.4s, v3.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[1]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[1]			 \n"

//		"	ldr		q6, [x12], #16					 \n"

		"	fmla	v28.4s, v10.4s, v3.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v29.4s, v11.4s, v3.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL4x16_K2          		         \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[2]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[2]			 \n"

		"	prfm	PLDL1KEEP, [x14, #64]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[2]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[2]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[2]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[2]			 \n"

//		"	ldr		q7, [x13], #16					 \n"  //

		"	fmla	v28.4s, v10.4s, v3.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v29.4s, v11.4s, v3.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL4x16_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x14, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[3]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[3]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v19.4s, v9.4s, v3.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v26.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[3]			 \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmla	v28.4s, v10.4s, v3.s[3]			 \n"
		"	ldr		q10, [x24], #16				 	 \n"
		"	fmla	v29.4s, v11.4s, v3.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		"	ldr		q3, [x14], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL4x16_END_K   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[3]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v9.4s, v3.s[3]			 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[3]			 \n"

		"	fmla	v28.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v29.4s, v11.4s, v3.s[3]			 \n"

		".endm									 	 \n"

		".macro	M4_ADD_C 							 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"

		"	ldp		q0, q1, [x25]					 \n"

		"	fadd	v12.4s, v12.4s, v0.4s			 \n"
		"	ldp		q2, q3, [x25, #32]				 \n"
		"	fadd	v13.4s, v13.4s, v1.4s			 \n"

		"	fadd	v22.4s, v2.4s, v22.4s			 \n"
		"	ldp		q4, q5, [x26]					 \n"
		"	fadd	v23.4s, v3.4s, v23.4s		 	 \n"

		"	fadd	v14.4s, v4.4s, v14.4s			 \n"
		"	ldp		q6, q7, [x26, #32]				 \n"
		"	fadd	v15.4s, v5.4s, v15.4s			 \n"

		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"

		"	fadd	v24.4s, v6.4s, v24.4s			 \n"
		"	ldp		q8, q9, [x27]					 \n"
		"	fadd 	v25.4s, v7.4s, v25.4s			 \n"

		"	fadd	v16.4s, v8.4s, v16.4s			 \n"
		"	ldp		q10, q11, [x27, #32]			 \n"
		"	fadd	v17.4s, v9.4s, v17.4s			 \n"

		"	fadd	v26.4s, v10.4s, v26.4s			 \n"
		"	ldp		q0, q1, [x28]				  	 \n"
		"	fadd	v27.4s, v11.4s, v27.4s			 \n"

		"	fadd	v18.4s, v0.4s, v18.4s			 \n"
		"	ldp		q2, q3, [x28, #32]				 \n"
		"	fadd	v19.4s, v1.4s, v19.4s			 \n"

		"	fadd	v28.4s, v2.4s, v28.4s			 \n"
		"	fadd	v29.4s, v3.4s, v29.4s			 \n"

		".endm 										 \n"

		".macro SAVE4x16							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	stp		q16, q17, [x27]					 \n"
		"	stp		q18, q19, [x28]					 \n"

		"	stp		q22, q23, [x25, #32]			 \n"
		"	stp		q24, q25, [x26, #32]			 \n"
		"	stp		q26, q27, [x27, #32]			 \n"
		"	stp		q28, q29, [x28, #32]			 \n"

		".endm 									 	 \n"

		".macro	KERNEL3x16_BEGIN_K					 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmul	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmul	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmul	v24.4s, v10.4s, v1.s[0]	         \n"
		"	fmul	v25.4s, v11.4s, v1.s[0]	         \n"

		"	fmul	v26.4s, v10.4s, v2.s[0]	         \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmul	v27.4s, v11.4s, v2.s[0]	         \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL3x16_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[0]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[0]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v27.4s, v11.4s, v2.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm   									 \n"

		".macro KERNEL3x16_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v17.4s, v9.4s, v2.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v27.4s, v11.4s, v2.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL3x16_K2          		         \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[2]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[2]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[2]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[2]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v27.4s, v11.4s, v2.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL3x16_K3   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[3]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v17.4s, v9.4s, v2.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v26.4s, v10.4s, v2.s[3]			 \n"
		"	ldr		q10, [x24], #16				 	 \n"
		"	fmla	v27.4s, v11.4s, v2.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		"	ldr		q2, [x13], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL3x16_END_K   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v9.4s, v2.s[3]			 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v26.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v27.4s, v11.4s, v2.s[3]			 \n"

		".endm									 	 \n"

		".macro	M3_ADD_C 							 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"

		"	ldp		q0, q1, [x25]					 \n"

		"	fadd	v12.4s, v12.4s, v0.4s			 \n"
		"	ldp		q2, q3, [x25, #32]				 \n"
		"	fadd	v13.4s, v13.4s, v1.4s			 \n"

		"	fadd	v22.4s, v2.4s, v22.4s			 \n"
		"	ldp		q4, q5, [x26]					 \n"
		"	fadd	v23.4s, v3.4s, v23.4s		 	 \n"

		"	fadd	v14.4s, v4.4s, v14.4s			 \n"
		"	ldp		q6, q7, [x26, #32]				 \n"
		"	fadd	v15.4s, v5.4s, v15.4s			 \n"

		"	prfm	PLDL1KEEP, [x27, #64]			 \n"

		"	fadd	v24.4s, v6.4s, v24.4s			 \n"
		"	ldp		q8, q9, [x27]					 \n"
		"	fadd 	v25.4s, v7.4s, v25.4s			 \n"

		"	fadd	v16.4s, v8.4s, v16.4s			 \n"
		"	ldp		q10, q11, [x27, #32]			 \n"
		"	fadd	v17.4s, v9.4s, v17.4s			 \n"

		"	fadd	v26.4s, v10.4s, v26.4s			 \n"
		"	fadd	v27.4s, v11.4s, v27.4s			 \n"

		".endm 										 \n"

		".macro SAVE3x16							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	stp		q16, q17, [x27]					 \n"

		"	stp		q22, q23, [x25, #32]			 \n"
		"	stp		q24, q25, [x26, #32]			 \n"
		"	stp		q26, q27, [x27, #32]			 \n"

		".endm 									 	 \n"

		".macro	KERNEL2x16_BEGIN_K					 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmul	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmul	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmul	v24.4s, v10.4s, v1.s[0]	         \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmul	v25.4s, v11.4s, v1.s[0]	         \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL2x16_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[0]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[0]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v25.4s, v11.4s, v1.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm   									 \n"

		".macro KERNEL2x16_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v8.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v15.4s, v9.4s, v1.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v25.4s, v11.4s, v1.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL2x16_K2          		         \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[2]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[2]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v25.4s, v11.4s, v1.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL2x16_K3   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	ldr		q10, [x24], #16				 	 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		"	ldr		q1, [x12], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL2x16_END_K   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v9.4s, v1.s[3]			 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v24.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v25.4s, v11.4s, v1.s[3]			 \n"

		".endm									 	 \n"

		".macro	M2_ADD_C 							 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"

		"	ldp		q0, q1, [x25]					 \n"

		"	fadd	v12.4s, v12.4s, v0.4s			 \n"
		"	ldp		q2, q3, [x25, #32]				 \n"
		"	fadd	v13.4s, v13.4s, v1.4s			 \n"

		"	fadd	v22.4s, v2.4s, v22.4s			 \n"
		"	ldp		q4, q5, [x26]					 \n"
		"	fadd	v23.4s, v3.4s, v23.4s		 	 \n"

		"	fadd	v14.4s, v4.4s, v14.4s			 \n"
		"	ldp		q6, q7, [x26, #32]				 \n"
		"	fadd	v15.4s, v5.4s, v15.4s			 \n"

		"	fadd	v24.4s, v6.4s, v24.4s			 \n"
		"	fadd 	v25.4s, v7.4s, v25.4s			 \n"

		".endm 										 \n"

		".macro SAVE2x16							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"

		"	stp		q22, q23, [x25, #32]			 \n"
		"	stp		q24, q25, [x26, #32]			 \n"

		".endm 									 	 \n"

		".macro	KERNEL1x16_BEGIN_K					 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmul	v22.4s, v10.4s, v0.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmul	v23.4s, v11.4s, v0.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm 										 \n"

		".macro KERNEL1x16_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v22.4s, v10.4s, v0.s[0]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v23.4s, v11.4s, v0.s[0]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm   									 \n"

		".macro KERNEL1x16_K1        	             \n"

		"	fmla	v12.4s, v8.4s, v0.s[1]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v13.4s, v9.4s, v0.s[1]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v22.4s, v10.4s, v0.s[1]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v23.4s, v11.4s, v0.s[1]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL1x16_K2          		         \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"	fmla	v22.4s, v10.4s, v0.s[2]			 \n"
		"	ldr		q10, [x24], #16					 \n"
		"	fmla	v23.4s, v11.4s, v0.s[2]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		".endm									 	 \n"

		".macro KERNEL1x16_K3   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	ldr		q8, [x24], #16					 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"
		"	ldr		q9, [x24], #16					 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	ldr		q10, [x24], #16				 	 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"
		"	ldr		q11, [x24], #16					 \n"

		"	ldr		q0, [x11], #16					 \n"

		".endm									 	 \n"

		".macro	M1_ADD_C 							 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"

		"	ldp		q0, q1, [x25]					 \n"

		"	fadd	v12.4s, v12.4s, v0.4s			 \n"
		"	ldp		q2, q3, [x25, #32]				 \n"
		"	fadd	v13.4s, v13.4s, v1.4s			 \n"

		"	fadd	v22.4s, v2.4s, v22.4s			 \n"
		"	fadd	v23.4s, v3.4s, v23.4s		 	 \n"

		".endm 										 \n"

		".macro KERNEL1x16_END_K   	                 \n"

		"	fmla	v12.4s, v8.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[3]			 \n"

		"	fmla	v22.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v23.4s, v11.4s, v0.s[3]			 \n"

		".endm									 	 \n"

		".macro SAVE1x16							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q22, q23, [x25, #32]			 \n"

		".endm 									 	 \n"

		".macro N8_SAVE5x8							 \n"

		"	subs	x21, x21, #1					 \n"

		"	add 	x10, x10, x5, lsl #4			 \n"
		"	add		x10, x10, x5, lsl #2 			 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	add		x25, x29, x6					 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	add		x26, x25, x6					 \n"
		"	stp		q16, q17, [x27]					 \n"
		"	add		x27, x26, x6					 \n"
		"	stp		q18, q19, [x28]					 \n"
		"	add		x28, x27, x6					 \n"
		"	stp		q20, q21, [x29]					 \n"
		"	add		x29, x28, x6					 \n"

		".endm 									 	 \n"

		".macro	N8_KERNEL5x8_BEGIN_K  				 \n"

		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"
		"	prfm	PLDL1KEEP, [x15, #64]			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"

		"	ldr		q4, [x15], #16				 	 \n"

		"	fmul	v18.4s, v8.4s, v3.s[0]	         \n"
		"	fmul	v19.4s, v9.4s, v3.s[0]	         \n"

		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v20.4s, v8.4s, v4.s[0]			 \n"
		"	fmul	v21.4s, v9.4s, v4.s[0]			 \n"

		".endm 										 \n"

		".macro N8_KERNEL5x8_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[0]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[0]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[0]			 \n"
		"	fmla	v21.4s, v9.4s, v4.s[0]			 \n"

		".endm   									 \n"

		".macro N8_KERNEL5x8_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[1]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[1]			 \n"

		"	fmla	v18.4s, v10.4s, v3.s[1]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[1]			 \n"

		"	fmla	v20.4s, v10.4s, v4.s[1]			 \n"
		"	fmla	v21.4s, v11.4s, v4.s[1]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL5x8_K2          		     \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[2]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[2]			 \n"

		"	fmla	v20.4s, v8.4s, v4.s[2]			 \n"
		"	fmla	v21.4s, v9.4s, v4.s[2]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL5x8_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x15, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"
		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmla	v18.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[3]			 \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmla	v20.4s, v10.4s, v4.s[3]			 \n"
		"	fmla	v21.4s, v11.4s, v4.s[3]			 \n"

		"	ldr		q4, [x15], #16					 \n"

		".endm									 	 \n"

		".macro N8_KERNEL5x8_END_K   	             \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		"	fmla	v18.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[3]			 \n"

		"	fmla	v20.4s, v10.4s, v4.s[3]			 \n"
		"	fmla	v21.4s, v11.4s, v4.s[3]			 \n"

		".endm									 	 \n"

		".macro 	N8_KERNEL4x8_BEGIN_K			 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	ldr		q3, [x14], #16					 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v18.4s, v8.4s, v3.s[0]	         \n"
		"	fmul	v19.4s, v9.4s, v3.s[0]	         \n"

		".endm 										 \n"

		".macro N8_KERNEL4x8_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[0]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[0]			 \n"

		".endm   									 \n"

		".macro N8_KERNEL4x8_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[1]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[1]			 \n"

		"	fmla	v18.4s, v10.4s, v3.s[1]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[1]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL4x8_K2          		     \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"

		"	fmla	v18.4s, v8.4s, v3.s[2]			 \n"
		"	fmla 	v19.4s, v9.4s, v3.s[2]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL4x8_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x15, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"
		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmla	v18.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[3]			 \n"

		"	ldr		q3, [x14], #16					 \n"

		".endm									 	 \n"

		".macro N8_KERNEL4x8_END_K   	             \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		"	fmla	v18.4s, v10.4s, v3.s[3]			 \n"
		"	fmla	v19.4s, v11.4s, v3.s[3]			 \n"

		".endm									 	 \n"

		".macro N8_SAVE4x8 							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	stp		q16, q17, [x27]					 \n"
		"	stp		q18, q19, [x28]					 \n"

		".endm 										 \n"

		".macro 	N8_KERNEL3x8_BEGIN_K			 \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"
		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x14, #64] 			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	ldr		q2, [x13], #16					 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v16.4s, v8.4s, v2.s[0]	         \n"
		"	fmul	v17.4s, v9.4s, v2.s[0]	         \n"

		".endm 										 \n"

		".macro N8_KERNEL3x8_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[0]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[0]			 \n"

		".endm   									 \n"

		".macro N8_KERNEL3x8_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[1]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[1]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[1]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[1]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL3x8_K2          		     \n"

		"   prfm    PLDL1KEEP, [x13, #64]       	 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		"	fmla	v16.4s, v8.4s, v2.s[2]			 \n"
		"	fmla 	v17.4s, v9.4s, v2.s[2]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL3x8_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x15, #64]       	 \n"
		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"
		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		"	ldr		q2, [x13], #16					 \n"

		".endm									 	 \n"

		".macro N8_KERNEL3x8_END_K   	             \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	fmla	v16.4s, v10.4s, v2.s[3]			 \n"
		"	fmla	v17.4s, v11.4s, v2.s[3]			 \n"

		".endm									 	 \n"

		".macro N8_SAVE3x8 							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"
		"	stp		q16, q17, [x27]					 \n"

		".endm 										 \n"

		".macro  N8_KERNEL2x8_BEGIN_K			     \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldr		q1, [x12], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"
		"	prfm	PLDL1KEEP, [x12, #64] 			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v14.4s, v8.4s, v1.s[0]	         \n"
		"	fmul	v15.4s, v9.4s, v1.s[0]	         \n"

		".endm 										 \n"

		".macro N8_KERNEL2x8_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[0]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[0]			 \n"

		".endm   									 \n"

		".macro N8_KERNEL2x8_K1        	             \n"

		"   prfm    PLDL1KEEP, [x12, #64]       	 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v12.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[1]			 \n"

		"	fmla	v14.4s, v10.4s, v1.s[1]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[1]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL2x8_K2          		     \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		"	fmla	v14.4s, v8.4s, v1.s[2]			 \n"
		"	fmla 	v15.4s, v9.4s, v1.s[2]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL2x8_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		"	ldr		q1, [x12], #16					 \n"

		".endm									 	 \n"

		".macro N8_KERNEL2x8_END_K   	             \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	fmla	v14.4s, v10.4s, v1.s[3]			 \n"
		"	fmla	v15.4s, v11.4s, v1.s[3]			 \n"

		".endm									 	 \n"

		".macro N8_SAVE2x8 							 \n"

		"	stp		q12, q13, [x25]			 		 \n"
		"	stp		q14, q15, [x26]			 	     \n"

		".endm 										 \n"

		".macro  N8_KERNEL1x8_BEGIN_K			     \n"

		"										 	 \n"
		"	ldr		q0, [x11], #16					 \n"
		"	ldp		q8, q9, [x24], #32			     \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"	prfm	PLDL1KEEP, [x24, #128]			 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmul	v12.4s, v8.4s, v0.s[0]	         \n"
		"	fmul	v13.4s, v9.4s, v0.s[0]	         \n"

		".endm 										 \n"

		".macro N8_KERNEL1x8_K0                   	 \n"

		"   prfm    PLDL1KEEP, [x11, #64]       	 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v12.4s, v8.4s, v0.s[0]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[0]			 \n"
		".endm   									 \n"

		".macro N8_KERNEL1x8_K1        	             \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v12.4s, v10.4s, v0.s[1]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[1]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL1x8_K2          		     \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q10, q11, [x24], #32			 \n"

		"	fmla	v12.4s, v8.4s, v0.s[2]			 \n"
		"	fmla	v13.4s, v9.4s, v0.s[2]			 \n"

		".endm									 	 \n"

		".macro N8_KERNEL1x8_K3   	                 \n"

		"   prfm    PLDL1KEEP, [x24, #128]       	 \n"
		"	ldp		q8, q9, [x24], #32				 \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		"	ldr		q0, [x11], #16					 \n"

		".endm									 	 \n"

		".macro N8_KERNEL1x8_END_K   	             \n"

		"	fmla	v12.4s, v10.4s, v0.s[3]			 \n"
		"	fmla	v13.4s, v11.4s, v0.s[3]			 \n"

		".endm									 	 \n"

		".macro N8_SAVE1x8 							 \n"

		"	stp		q12, q13, [x25]			 		 \n"

		".endm 										 \n"

//----------------------------------------------------

		"SMM:									 	 \n"

		"	ldr		x0, %[C]						 \n"
		"	ldr		x1, %[A]						 \n"
		"	ldr		x2, %[B]						 \n"

		"	ldr		x3, %[M]						 \n"
		"	ldr		x4, %[N]						 \n"
		"	ldr		x5, %[K]						 \n"
//		"	ldr		x7, %[temp]						 \n"
		"	ldr		x9, %[LN] 						 \n"
		"	ldr		x6, %[LK]						 \n"

		"	ldr		x19, %[SB]						 \n"
		"	ldr		x8, %[k_tag] 					 \n"
//		"	lsl		x6, x4, #2						 \n"  //sizeof(N)

		"	add		sp, sp, #-32					 \n"

		"	mov		x23, #5						 	 \n"
//		"	lsl		x8, x5, #2						 \n"  //sizeof(K)

		"   prfm    PLDL1KEEP, [x1]             	 \n"
		"   prfm    PLDL1KEEP, [x2]             	 \n"

		"	udiv	x30, x3, x23				 	 \n"	// M / 5
		"	lsr		x20, x4, #4					 	 \n"	// N / 16

		"	msub	x23, x30, x23, x3 				 \n"	// M % 5
		"	str 	x23, [sp] 						 \n"
		"	mov 	x23, x19 						 \n"    //SB
		"	str  	x8, [sp, #16]					 \n"

//		"	cmp		x20, #0							 \n"
//		"	beq		BEGIN_N8						 \n"

//-----------------------------------------------N16
		"BEGIN_N16:								 	 \n"

		"	mov 	x25, x0   						 \n"   //C1*
		"	add		x26, x25, x9, lsl #2 			 \n"   //C2*
		"	add 	x27, x25, x9, lsl #3 			 \n"   //C3*
		" 	add 	x28, x26, x9, lsl #3 			 \n"   //C4*
		"	add		x29, x27, x9, lsl #3 			 \n"   //C5*

		"	mov		x10, x1   						 \n"
		"	mov		x21, x30  						 \n"
		"	mov		x24, x23						 \n"   //还原SB的地址

		"	mov		x7, #4							 \n"   //4次B的循环
		"	mov		x8, #16							 \n"

		"BEGIN_PACKB:								 \n"

		"	mov		x16, x2						 	 \n"   //B0*
		"	add		x17, x16, x6, lsl #2 			 \n"
		"	add		x18, x17, x6, lsl #2 			 \n"
		"	add		x19, x18, x6, lsl #2 			 \n"

		"	mov		x11, x10						 \n"    //A0*
		"	add		x12, x11, x6, lsl #2			 \n"    //A1*
		"	add		x13, x12, x6, lsl #2			 \n"	//A2*
		"	add		x14, x13, x6, lsl #2			 \n"	//A3*
		"	add		x15, x14, x6, lsl #2 			 \n"	//A4*

		"	prfm    PLDL1KEEP, [x16, #64]			 \n"
		"	prfm    PLDL1KEEP, [x17, #64]			 \n"
		"	prfm    PLDL1KEEP, [x18, #64]			 \n"
		"	prfm    PLDL1KEEP, [x19, #64]			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"
		"	prfm    PLDL1KEEP, [x14, #64]			 \n"
		"	prfm    PLDL1KEEP, [x15, #64]			 \n"

		"PACK_Body_K:								 \n"

		"	lsr		x22, x5, #3 					 \n"     // K / 8

		"	PACK_KERNEL5x4_BEGIN_K				 	 \n"

		"	subs	x22, x22, #1		     		 \n"
		"	b 		PACK_K1_7						 \n"

		"PACK_K:									 \n"

		"	PACK_KERNEL5x4_K0						 \n"
		"PACK_K1_7:									 \n"

		"	beq		PACK_Edge_K						 \n"

		"	PACK_KERNEL5x4_K1						 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		PACK_K			 				 \n"

		"PACK_Edge_K:							     \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"
		"	prfm	PLDL1KEEP, [x29, #64]			 \n"

		"	PACK_KERNEL5x4_END_K			         \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		PACKB_SAVE1 					 \n"
		"	PACKB_ADD_C 							 \n"
		"PACKB_SAVE1: 								 \n"
		"	SAVE5x4								 	 \n"
		"	mov		x8, #16							 \n"

//		"	beq		END_PACKB						 \n"

		"	add		x23, x23, #16					 \n"
		"	mov		x24, x23 						 \n"
		"	add		x2, x2, x6, lsl #4				 \n"

		"	bgt  	BEGIN_PACKB					 	 \n"

		"END_PACKB:									 \n"

		"	sub		x23, x23, x8, lsl #2			 \n"    //还原SB的索引

		"	sub 	x29, x29, x8, lsl #2			 \n"    //C矩阵的偏移量还原
		"	add		x25, x29, x9, lsl #2			 \n"
		"	add		x26, x25, x9, lsl #2			 \n"
		"	add		x27, x26, x9, lsl #2			 \n"
		"	add		x28, x27, x9, lsl #2			 \n"
		"	add		x29, x28, x9, lsl #2 			 \n"

		"	add		x10, x10, x6, lsl #4			 \n"
		"	add 	x10, x10, x6, lsl #2 			 \n"	// A + 5行

		"	cmp		x8, #16							 \n"
		"	bne 	N8_END_PACKB					 \n"

		"	subs	x21, x21, #1					 \n"
		"	beq		END_M5							 \n"

		//---------------------------------------------------

		"BEGIN_M5:									 \n"

		"	mov		x24, x23	     				 \n"

		"	mov		x11, x10						 \n"
		"	add		x12, x11, x6, lsl #2			 \n"
		"	add		x13, x12, x6, lsl #2			 \n"
		"	add		x14, x13, x6, lsl #2			 \n"
		"	add		x15, x14, x6, lsl #2 			 \n"

		"	prfm	PLDL1KEEP, [x24, #512]			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"
		"	prfm    PLDL1KEEP, [x14, #64]			 \n"
		"	prfm    PLDL1KEEP, [x15, #64]			 \n"

		"Body_K:								 	 \n"

		"	lsr		x22, x5, #3 					 \n"     // K / 8
		"	KERNEL5x16_BEGIN_K						 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b K1_7									 \n"

		"Main_K:									 \n"

		"	KERNEL5x16_K0						 	 \n"
		"K1_7:										 \n"

		"	KERNEL5x16_K1							 \n"
		"	KERNEL5x16_K2							 \n"
		"	KERNEL5x16_K3							 \n"
		"	KERNEL5x16_K4							 \n"
		"	KERNEL5x16_K5							 \n"
		"	KERNEL5x16_K6							 \n"

		"	beq		Edge_K							 \n"

		"	KERNEL5x16_K7							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		Main_K			 				 \n"

		"Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"
		"	prfm	PLDL1KEEP, [x29, #64]			 \n"

		"	KERNEL5x16_END_K						 \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		M5_SAVE1 						 \n"
		"	ADD_C 								 	 \n"
		"M5_SAVE1: 									 \n"
		"	SAVE5x16								 \n"

//		"	SAVE5x16								 \n"
		"	bgt   	BEGIN_M5						 \n"

		"END_M5:									 \n"

		"	mov		x24, x23	     				 \n"  //B0*

		"	ldr		x21, [sp]					 	 \n"
		"	cmp		x21, #4							 \n"
		"	bne 	BEGIN_M3 						 \n"

	//--------------------------------------------------------
		"BEGIN_M4:						 			 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x6, lsl #2 		     \n"
		"	add		x13, x12, x6, lsl #2 			 \n"
		"	add		x14, x13, x6, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"
		"	prfm    PLDL1KEEP, [x14, #64]			 \n"

		"M4_Body_K:								 	 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	KERNEL4x16_BEGIN_K						 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		M4_K1_3							 \n"

		"M4_Main_K:									 \n"

		"	KERNEL4x16_K0						 	 \n"
		"M4_K1_3:									 \n"

		"	KERNEL4x16_K1							 \n"
		"	KERNEL4x16_K2							 \n"

		"	beq		M4_Edge_K						 \n"

		"	KERNEL4x16_K3							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		M4_Main_K			 			 \n"

		"M4_Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"

		"	KERNEL4x16_END_K						 \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		M4_SAVE1 						 \n"
		"	M4_ADD_C 								 \n"
		"M4_SAVE1: 									 \n"
		"	SAVE4x16								 \n"

		"	b 		END_M 							 \n"

		//--------------------------------------------------------

		"BEGIN_M3:			 						 \n"
		"	cmp		x21, #3							 \n"
		"	bne 	BEGIN_M2 						 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x6, lsl #2 			 \n"
		"	add		x13, x12, x6, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"

		"M3_Body_K:								 	 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	KERNEL3x16_BEGIN_K						 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		M3_K1_3							 \n"

		"M3_Main_K:									 \n"

		"	KERNEL3x16_K0						 	 \n"
		"M3_K1_3:									 \n"

		"	KERNEL3x16_K1							 \n"
		"	KERNEL3x16_K2							 \n"

		"	beq		M3_Edge_K						 \n"

		"	KERNEL3x16_K3							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		M3_Main_K			 			 \n"

		"M3_Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"

		"	KERNEL3x16_END_K						 \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		M3_SAVE1 						 \n"
		"	M3_ADD_C 								 \n"
		"M3_SAVE1: 									 \n"
		"	SAVE3x16								 \n"

		"	b 		END_M 							 \n"

//----------------------------------------------------------
		"BEGIN_M2:			 						 \n"

		"	cmp		x21, #2							 \n"
		"	bne 	BEGIN_M1 						 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x6, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"

		"M2_Body_K:								 	 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	KERNEL2x16_BEGIN_K						 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		M2_K1_3							 \n"

		"M2_Main_K:									 \n"

		"	KERNEL2x16_K0						 	 \n"
		"M2_K1_3:									 \n"

		"	KERNEL2x16_K1							 \n"
		"	KERNEL2x16_K2							 \n"

		"	beq		M2_Edge_K						 \n"

		"	KERNEL2x16_K3							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		M2_Main_K			 			 \n"

		"M2_Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"

		"	KERNEL2x16_END_K						 \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		M2_SAVE1 						 \n"
		"	M2_ADD_C 								 \n"
		"M2_SAVE1: 									 \n"
		"	SAVE2x16								 \n"

		"	b 		END_M 							 \n"

//---------------------------------------------------------

		"BEGIN_M1:									 \n"

		"	cmp		x21, #1							 \n"
		"	bne 	END_M 						 	 \n"

		"	mov		x11, x10 						 \n"
		"	prfm    PLDL1KEEP, [x11, #64]			 \n"

		"M1_Body_K:								 	 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	KERNEL1x16_BEGIN_K						 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		M1_K1_3							 \n"

		"M1_Main_K:									 \n"

		"	KERNEL1x16_K0						 	 \n"
		"M1_K1_3:									 \n"

		"	KERNEL1x16_K1							 \n"
		"	KERNEL1x16_K2							 \n"

		"	beq		M1_Edge_K						 \n"

		"	KERNEL1x16_K3							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		M1_Main_K			 			 \n"

		"M1_Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"

		"	KERNEL1x16_END_K						 \n"

		"	ldr		x8, [sp, #16]					 \n"
		"	cmp		x8, #0							 \n"
		"	beq		M1_SAVE1 						 \n"
		"	M1_ADD_C 								 \n"
		"M1_SAVE1: 									 \n"

		"	SAVE1x16								 \n"

		"END_M:										 \n"

		"	subs	x20, x20, #1					 \n"

		"	add		x16, x16, #64					 \n"
		"	add		x0, x0, #64						 \n"

		"	bgt		BEGIN_N16						 \n"

		"END_N16:									 \n"

		"	ands	x20, x4, #15					 \n"
		"	beq		END_N 							 \n"

	    "N8_END_PACKB:	    						 \n"
	/*

		"BEGIN_N8:									\n"

//----------------------------------------------------------N8
		"N8_BEGIN_PACKB:							 \n"

		"	mov 	x25, x0   						 \n"   //C1*
		"	add		x26, x25, x6 					 \n"   //C2*
		"	add 	x27, x25, x6, lsl #1 			 \n"   //C3*
		" 	add 	x28, x26, x6, lsl #1 			 \n"   //C4*
		"	add		x29, x27, x6, lsl #1 			 \n"   //C5*

		"	mov		x10, x1   						 \n"   //A0*
		"	mov		x21, x30  						 \n"   // M / 5
		"	mov		x7, #2							 \n"
		"	mov		x24, x23						 \n"

		"	mov		x8, #8 							 \n"

		"	b 		BEGIN_PACKB 					 \n"

		"N8_END_PACKB:								 \n"

		"	subs	x21, x21, #1					 \n"
		"	beq		N8_END_M5						 \n"

//--------------------------------------------------N8M5
		"N8_BEGIN_M5:								 \n"

		"	mov		x24, x23	     				 \n"

		"	mov		x11, x10						 \n"
		"	add		x12, x11, x5, lsl #2			 \n"
		"	add		x13, x12, x5, lsl #2			 \n"
		"	add		x14, x13, x5, lsl #2			 \n"
		"	add		x15, x14, x5, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"
		"	prfm    PLDL1KEEP, [x14, #64]			 \n"
		"	prfm    PLDL1KEEP, [x15, #64]			 \n"

		"N8_Body_K:								 	 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	N8_KERNEL5x8_BEGIN_K					 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		N8_K1_3							 \n"

		"N8_Main_K:									 \n"

		"	N8_KERNEL5x8_K0						 	 \n"
		"N8_K1_3:									 \n"

		"	N8_KERNEL5x8_K1							 \n"
		"	N8_KERNEL5x8_K2							 \n"

		"	beq		N8_Edge_K						 \n"

		"	N8_KERNEL5x8_K3							 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		N8_Main_K			 			 \n"

		"N8_Edge_K:									 \n"

		"	prfm	PLDL1KEEP, [x25, #64]			 \n"
		"	prfm	PLDL1KEEP, [x26, #64]			 \n"
		"	prfm	PLDL1KEEP, [x27, #64]			 \n"
		"	prfm	PLDL1KEEP, [x28, #64]			 \n"
		"	prfm	PLDL1KEEP, [x29, #64]			 \n"

		"	N8_KERNEL5x8_END_K						 \n"

		"	N8_SAVE5x8								 \n"
		"	bgt   	N8_BEGIN_M5					 	 \n"

//---------------------------------------------------------

		"N8_END_M5:									 \n"

		"	mov		x24, x23	     				 \n"  //B0*

		"	ldr		x21, [sp]						 \n"
		"	cmp		x21, #4							 \n"
		"	bne 	N8_BEGIN_M3 					 \n"

	//--------------------------------------------------------
		"N8_BEGIN_M4:						 		 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x5, lsl #2 			 \n"
		"	add		x13, x12, x5, lsl #2 			 \n"
		"	add		x14, x13, x5, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"
		"	prfm    PLDL1KEEP, [x14, #64]			 \n"

		"N8_M4_Body_K:								 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	N8_KERNEL4x8_BEGIN_K					 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		N8_M4_K1_3						 \n"

		"N8_M4_Main_K:								 \n"

		"	N8_KERNEL4x8_K0						 	 \n"
		"N8_M4_K1_3:								 \n"

		"	N8_KERNEL4x8_K1						 	 \n"
		"	N8_KERNEL4x8_K2						 	 \n"

		"	beq		N8_M4_Edge_K					 \n"

		"	N8_KERNEL4x8_K3						 	 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		N8_M4_Main_K			 		 \n"

		"N8_M4_Edge_K:								 \n"

		"	prfm	PLDL1KEEP, [x25, #32]			 \n"
		"	prfm	PLDL1KEEP, [x26, #32]			 \n"
		"	prfm	PLDL1KEEP, [x27, #32]			 \n"
		"	prfm	PLDL1KEEP, [x28, #32]			 \n"

		"	N8_KERNEL4x8_END_K						 \n"
		"	N8_SAVE4x8								 \n"

		"	b 		N8_END_M 						 \n"

//------------------------------------------------------------------

		"N8_BEGIN_M3: 								 \n"

		"	cmp		x21, #3							 \n"
		"	bne 	N8_BEGIN_M2 					 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x5, lsl #2 			 \n"
		"	add		x13, x12, x5, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"
		"	prfm    PLDL1KEEP, [x13, #64]			 \n"

		"N8_M3_Body_K:								 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	N8_KERNEL3x8_BEGIN_K					 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		N8_M3_K1_3						 \n"

		"N8_M3_Main_K:								 \n"

		"	N8_KERNEL3x8_K0						 	 \n"
		"N8_M3_K1_3:								 \n"

		"	N8_KERNEL3x8_K1						 	 \n"
		"	N8_KERNEL3x8_K2						 	 \n"

		"	beq		N8_M3_Edge_K					 \n"

		"	N8_KERNEL3x8_K3						 	 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		N8_M3_Main_K			 		 \n"

		"N8_M3_Edge_K:								 \n"

		"	prfm	PLDL1KEEP, [x25, #32]			 \n"
		"	prfm	PLDL1KEEP, [x26, #32]			 \n"
		"	prfm	PLDL1KEEP, [x27, #32]			 \n"

		"	N8_KERNEL3x8_END_K						 \n"
		"	N8_SAVE3x8								 \n"

		"	b 		N8_END_M 						 \n"

//--------------------------------------------------------

		"N8_BEGIN_M2:								 \n"

		"	cmp		x21, #2							 \n"
		"	bne 	N8_BEGIN_M1 					 \n"

		"	mov		x11, x10 						 \n"
		"	add		x12, x11, x5, lsl #2 			 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"
		"	prfm    PLDL1KEEP, [x12, #64]			 \n"

		"N8_M2_Body_K:								 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	N8_KERNEL2x8_BEGIN_K					 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		N8_M2_K1_3						 \n"

		"N8_M2_Main_K:								 \n"

		"	N8_KERNEL2x8_K0						 	 \n"
		"N8_M2_K1_3:								 \n"

		"	N8_KERNEL2x8_K1						 	 \n"
		"	N8_KERNEL2x8_K2						 	 \n"

		"	beq		N8_M2_Edge_K					 \n"

		"	N8_KERNEL2x8_K3						 	 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		N8_M2_Main_K			 		 \n"

		"N8_M2_Edge_K:								 \n"

		"	prfm	PLDL1KEEP, [x25, #32]			 \n"
		"	prfm	PLDL1KEEP, [x26, #32]			 \n"

		"	N8_KERNEL2x8_END_K						 \n"
		"	N8_SAVE2x8								 \n"

		"	b 		N8_END_M 						 \n"

//-----------------------------------------------------
		"N8_BEGIN_M1:								 \n"

		"	cmp		x21, #1							 \n"
		"	bne 	N8_END_M 					 	 \n"

		"	mov		x11, x10 						 \n"

		"	prfm    PLDL1KEEP, [x11, #64]			 \n"

		"N8_M1_Body_K:								 \n"

		"	lsr		x22, x5, #2 					 \n"     // K / 8
		"	N8_KERNEL1x8_BEGIN_K					 \n"
		"	subs	x22, x22, #1		     		 \n"
		"	b 		N8_M1_K1_3						 \n"

		"N8_M1_Main_K:								 \n"

		"	N8_KERNEL1x8_K0						 	 \n"
		"N8_M1_K1_3:								 \n"

		"	N8_KERNEL1x8_K1						 	 \n"
		"	N8_KERNEL1x8_K2						 	 \n"

		"	beq		N8_M1_Edge_K					 \n"

		"	N8_KERNEL1x8_K3						 	 \n"

		"	subs	x22, x22, #1			 		 \n"
		"	b 		N8_M1_Main_K			 		 \n"

		"N8_M1_Edge_K:								 \n"

		"	prfm	PLDL1KEEP, [x25, #32]			 \n"

		"	N8_KERNEL1x8_END_K						 \n"
		"	N8_SAVE1x8								 \n"

		"N8_END_M: 									 \n"
*/
		"END_N:									 	 \n"

		"	add		sp, sp, #32						 \n"

		:
		:
			[C] "m" (C),
			[A] "m" (A),
            [B] "m" (B),
            [M] "m" (M),
            [N] "m" (N),
            [K] "m" (K),
            [LN] "m" (LN),
            [LK] "m" (LK),
            [SB] "m" (SB),
            [k_tag] "m" (k_tag)
        :   "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",
        	"x9", "x10", "x11", "x12", "x13","x14", "x15", "x16",
        	"x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24","x25",
        	"x26", "x27", "x28", "x29", "x30",
            "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
            "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
            "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
            "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"

		);
}

void Small_SGEMM(float *C, float *A, float *B, long M, long N,
					long K, long *temp, long LD)
{

	omp_set_num_threads(num);
	long i, k, kc;
	long LK = K;
    long GEMM_K = 328;
    void *ptr;
    posix_memalign(&ptr, 64, num * GEMM_K * 16  *sizeof( float ));
    float *SSB = (float *)ptr;

	#   pragma omp parallel for private(k,kc)
	for(i=0; i< LD; i= i + N)
	{
		float *CC = C + i ;

		for(k = 0; k < LK; k = k + kc)
		{
			kc = GEMM_K;
			if(LK - k < GEMM_K)
				kc = LK - k;
			float * AA = A + k;
			float *BB = B + i * LK + k ;

			SMM(CC, AA, BB, M, N, kc, temp, LD, LK,
				&SSB[i/N * GEMM_K * 16], k);
		}
	}

	free(SSB);

}

int main()
{

	openblas_set_num_threads(num);

	int i,j,k;
	int loop=5;
	long M, N, K;
	double start, cost;
	int flag =0 ;
	void *ptr, *ptr1;
	long temp =-1;
	int pc;

	FILE *fp;
  	if( (fp=fopen("NSMM_P_FT.txt","w")) == NULL )
  	{
    	puts("Fail to open file!");
    	exit(0);
  	}

	K = 576;
	for(j = 5; j < 6;j++)
	{
                M = 262;
                N = 50176;
//		K = 4992 + 8 * j;
		double ops = M *N *K * 1.0e-09 * 2;

		fprintf(fp, "%d %d %d", M,N,K);
		for(pc =0; pc < 5; pc++)
		{
			posix_memalign(&ptr1, 4096, M* K * sizeof( float ));
			posix_memalign(&ptr, 4096, K* N * sizeof( float ));
			float *A = (float *)ptr1;
			float *B = (float *)ptr;
		    float *C = ( float * ) malloc( M* N * sizeof( float ) );

		    random_matrix(M,K,A);
		    random_matrix(K,N,B);
		   // transpose(K, N, B);

		    for( i =0 ;i< 2; i++)
		    	Small_SGEMM(C, A, B, M, N/num, K, &temp, N);

			start = dclock();
			for( i= 0; i< loop ;i++)
				Small_SGEMM(C, A, B, M, N/num, K, &temp, N);
			cost =(dclock()-start)/loop;

			ops = M * N  * K * 1.0e-09 * 2;
			printf("\nN_SMM:  M= %d N=%d K=%d flops = %lf effic= %.3lf %\n",
					M, N, K, ops/cost, ops/cost/17.6 * 100/num);

			fprintf(fp, " %.3f", ops/cost);
		    free(A);
		    free(B);
		    free(C);
		}

		fprintf(fp, "\n");
	}

	fclose(fp);
    return 0;

}

max_stars_repo_name: AnonymousYWL/MYLIB

max_issues_repo_name: AnonymousYWL/LibShalom

max_forks_repo_name: AnonymousYWL/LibShalom


text:
#include <stdlib.h>
#if !defined(__APPLE__)
#include <malloc.h>
#endif
#include <stdio.h>
#include <assert.h>
#include <time.h>
#include <string.h>

#include <fftw3.h>

#include <gsl/gsl_errno.h>
#include <gsl/gsl_sf_erf.h>
#include <gsl/gsl_integration.h>
#include <gsl/gsl_spline.h>
#include <gsl/gsl_sf_gamma.h>
#include <gsl/gsl_sf_legendre.h>
#include <gsl/gsl_sf_bessel.h>
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_matrix.h>
#include <gsl/gsl_eigen.h>
#include <gsl/gsl_sf_expint.h>
#include <gsl/gsl_deriv.h>
#include <gsl/gsl_interp2d.h>
#include <gsl/gsl_spline2d.h>

#include "../cosmolike_core/theory/basics.c"
#include "../cosmolike_core/theory/structs.c"
#include "../cosmolike_core/theory/parameters.c"
#include "../cosmolike_core/emu17/P_cb/emu.c"
#include "../cosmolike_core/theory/recompute.c"
#include "../cosmolike_core/theory/cosmo3D.c"
#include "../cosmolike_core/theory/redshift_spline.c"
#include "../cosmolike_core/theory/halo.c"
#include "../cosmolike_core/theory/HOD.c"
#include "../cosmolike_core/theory/pt.c"
#include "../cosmolike_core/theory/cosmo2D_fourier.c"
#include "../cosmolike_core/theory/IA.c"
#include "../cosmolike_core/theory/cluster.c"
#include "../cosmolike_core/theory/BAO.c"
#include "../cosmolike_core/theory/external_prior.c"
#include "../cosmolike_core/theory/GRS.c"
#include "like_grs.c"

double log_like_wrapper(input_cosmo_params ic, input_nuisance_params_grs in);

double log_like_wrapper(input_cosmo_params ic, input_nuisance_params_grs in)
{
	// printf("%le %le %le %le %le %le %le\n",ic.omega_m, ic.sigma_8, ic.n_s, ic.w0, ic.wa, ic.omega_b, ic.h0);
	// printf("%le %le %le %le %le %le %le\n",in.grsbias[0], in.grsbias[1], in.grsbias[2], in.grsbias[3],in.grsbias[4], in.grsbias[5], in.grsbias[6]);
	//printf("%le %le %le %le %le %le %le %le %le %le\n",in.grssigmap[0], in.grssigmap[1], in.grssigmap[2], in.grssigmap[3], in.grssigmap[4], in.grssigmap[5], in.grssigmap[6],in.grssigmaz, in.grspshot, in.grskstar);
  double like = log_like_GRS(ic.omega_m, ic.sigma_8, ic.n_s, ic.w0, ic.wa, ic.omega_b, ic.h0, ic.MGSigma, ic.MGmu,in.grsbias[0], in.grsbias[1], in.grsbias[2], in.grsbias[3],in.grsbias[4], in.grsbias[5], in.grsbias[6], in.grssigmap[0], in.grssigmap[1], in.grssigmap[2], in.grssigmap[3], in.grssigmap[4], in.grssigmap[5], in.grssigmap[6],in.grssigmaz, in.grspshot, in.grskstar);

  return like;
}

int main(void){
	double res;
	init_GRS();
	res=log_like_GRS(0.3156,0.831,0.9645,-1.0,0.0,0.0491685,0.6727,0.0,0.0,1.538026692020565,1.862707210288686,2.213131761595241,2.617023657038295,2.975011712138650,3.376705680190931,3.725882076395691,290.,290.,290.,290.,290.,290.,290.,0.001,0.0,0.24);
	printf("loglike=%le\n",res);
	res=log_like_GRS(0.2325384451023471,1.0310887691615935,0.8595022066924793, -1.137588053600136,  0.14993639190238117,  0.05165678142188447, 0.8247432633482439,0.0,0.0,1.538026692020565,1.862707210288686,2.213131761595241,2.617023657038295,2.975011712138650,3.376705680190931,3.725882076395691,290.,290.,290.,290.,290.,290.,290.,0.001,0.0,0.24);
	printf("loglike=%le\n",res);
return 0;
}

//	double log_multi_like(double OMM, double S8, double NS, double W0,double WA, double OMB, double H0,
//	double B1, double B2, double B3, double B4, double B5, double B6, double B7,
//	 double SIGMAP1, double SIGMAP2, double SIGMAP3, double SIGMAP4, double SIGMAP5, double SIGMAP6, double SIGMAP7,
//	 double SIGMAZ, double PSHOT, double KSTAR)

// 	log_multi_like(0.315,0.831,0.965,-1.0,0.0,0.049,0.673,1.19,1.30,1.44,1.57,1.70,1.82,1.93,290.,290.,290.,290.,290.,290.,290.,0.001,0.0,0.094);
// 	double dk, dmu, *k, *mu, *data, *variance;
// 	int i;
// 	k = create_double_vector(0,GRS.N_k-1);
// 	mu = create_double_vector(0,GRS.N_mu-1);
// 	data = create_double_vector(0,GRS.N_z*GRS.N_k*GRS.N_mu-1);
// 	variance = create_double_vector(0,GRS.N_z*GRS.N_k*GRS.N_mu-1);
// 	dk = (GRS.k_max-GRS.k_min)/GRS.N_k;
// 	dmu= 1./GRS.N_mu;
// 	for (i =0; i < GRS.N_k; i++){
// 		// use linear spacing in k!
// 		k[i] = GRS.k_min +(i+0.5)*dk;
// 	}
// 	for (i =0; i < GRS.N_mu; i++){
// 		// use linear spacing in mu!
// 		mu[i] = (i+0.5)*dmu;
// 	}
// 	set_data_GRS(k,mu,data);
// 	set_variance_GRS(k,mu, dk, dmu, variance);
// 	return 0;
// }

/*int main(void){
	set_cosmological_parameters_to_Planck_WP();
	init_GRS_single_zbin(0.16,0.4);
	double z;
	GRS_gal.b_g[0] = 1.0;
	for (z = 0.1; z < 2.5; z+=0.05){
		GRS.z[0] = z;
		printf("%.3f  %e %e %e\n", z, 1./P_obs(0.14,0.6,0), 1./P_obs(0.5,0.6,0),1./P_obs(1.0,0.6,0));
	}
	return 0;
}
*/


text:
#pragma once

#include <gsl/gsl>
#include <type_traits>

#include "Exponentials.h"
#include "HyperionUtils/Concepts.h"
#include "HyperionUtils/Macros.h"
#include "HyperionUtils/OptionAndResult.h"

namespace hyperion::math {
	using utils::None;
	using utils::Option;
	using utils::Some;
	using utils::concepts::FloatingPoint;
	/// @brief The possible kinds of value interpolation possible by `Interpolator`
	enum class InterpolationType
	{
		Linear = 0,
		Logarithmic,
		Exponential
	};

	IGNORE_PADDING_START
	/// @brief Interpolates from a starting value to a final value in the way prescribed by `Type`
	/// @note `T` must be a floating point type
	/// @note The start value must be in [0, target value] and the target value must be > 0
	///
	/// @tparam T - The floating point type to perform operations in
	/// @tparam Type - The type of interpolation
	template<FloatingPoint T, InterpolationType Type = InterpolationType::Linear>
	class Interpolator {
	  public:
		static constexpr T DEFAULT_TARGET_VALUE = gsl::narrow_cast<T>(1.0);
		static constexpr T DEFAULT_INITIAL_VALUE = gsl::narrow_cast<T>(0.0);
		static constexpr T DEFAULT_TRANSITION_LENGTH = gsl::narrow_cast<T>(0.1);
		static constexpr size_t DEFAULT_SAMPLE_RATE = 44100ULL;

		/// @brief Creates a default `Interpolator`
		constexpr Interpolator() noexcept = default;

		/// @brief Creates an `Interpolator` with the given target value
		///
		/// @param targetValue - The target final value to interpolate to
		constexpr explicit Interpolator(T targetValue) noexcept
			: m_target_value(targetValue),
			  mLinearTransitionStep(
				  (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds)),
			  m_samples_to_transition(get_samples_to_transition()) {
		}

		/// @brief Creates an `Interpolator` with the given target value and sample rate
		///
		/// @param targetValue - The target final value to interpolate to
		/// @param sampleRate - The sample rate to process at
		constexpr explicit Interpolator(T targetValue, size_t sampleRate) noexcept
			: m_sample_rate(sampleRate), m_target_value(targetValue),
			  mLinearTransitionStep(
				  (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds)),
			  m_samples_to_transition(get_samples_to_transition()) {
		}

		/// @brief Creates an `Interpolator` with the given target and initial values, and sample
		/// rate
		///
		/// @param targetValue - The target final value to interpolate to
		/// @param initial_value - The initial value to start interpolation from
		/// @param sampleRate - The sample rate to process at
		constexpr Interpolator(T targetValue, T initial_value, size_t sampleRate) noexcept
			: m_sample_rate(sampleRate), m_target_value(targetValue),
			  m_current_value(initial_value), m_initial_value(m_current_value),
			  mLinearTransitionStep(
				  (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds)),
			  m_samples_to_transition(get_samples_to_transition()) {
		}

		/// @brief Creates an `Interpolator` with the given target and initial values, time to
		/// interpolate over, and sample rate
		///
		/// @param targetValue - The target final value to interpolate to
		/// @param initial_value - The initial value to start interpolation from
		/// @param transitionLengthSeconds - The transition time to interpolate over
		/// @param sampleRate - The sample rate to process at
		constexpr Interpolator(T targetValue,
							   T initial_value,
							   T transitionLengthSeconds,
							   size_t sampleRate) noexcept
			: m_sample_rate(sampleRate), m_target_value(targetValue),
			  m_current_value(initial_value), m_initial_value(m_current_value),
			  m_transition_lengthSeconds(transitionLengthSeconds),
			  mLinearTransitionStep(
				  (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds)),
			  m_samples_to_transition(get_samples_to_transition()) {
		}
		constexpr Interpolator(const Interpolator& interpolator) noexcept = default;
		constexpr Interpolator(Interpolator&& interpolator) noexcept = default;
		~Interpolator() noexcept = default;

		/// @brief Gets the next value in the interpolation sequence
		/// If `currentSample` is given, gets the value in the sequence for the given sample index
		/// @note This is only available for `Interpolator`s of type
		/// `InterpolationType::Logarithmic` or `InterpolationType::Exponential`
		/// @param currentSample - The (optional) sample in the sequence to get the value for
		///
		/// @return - The interpolation value
		inline auto get_next_value(Option<size_t> currentSample = None()) noexcept -> T {
			if(currentSample.is_some()) {
				m_current_transition_sample = currentSample.unwrap();
			}

			if(m_current_transition_sample <= m_samples_to_transition) {
				m_current_value = interpolate(m_current_transition_sample);
				m_current_transition_sample++;
			}

			return m_current_value;
		}

		/// @brief Resets the `Interpolator`.
		/// - If `initial_value` is given, the `Interpolator` will start at the given one instead of
		/// `DEFAULT_INITIAL_VALUE`
		/// - If `transitionLengthSeconds` is also given, the `Interpolator` will interpolate over
		/// that length of time instead of the current one
		///
		/// @param initial_value - The initial value to start interpolation from
		/// @param transitionLengthSeconds - The transition time to interpolate over
		inline auto
		reset(Option<T> initial_value, Option<T> transitionLengthSeconds) noexcept -> void {
			if(initial_value.is_some()) {
				m_current_value = initial_value.unwrap();
				m_initial_value = m_current_value;
			}
			else {
				m_current_value = DEFAULT_INITIAL_VALUE;
				m_initial_value = m_current_value;
			}

			if(transitionLengthSeconds.is_some()) {
				m_transition_lengthSeconds = transitionLengthSeconds.unwrap();
			}

			mLinearTransitionStep
				= (m_target_value - m_current_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds);
			m_samples_to_transition = get_samples_to_transition();
			m_current_transition_sample = 0ULL;
		}

		/// @brief Sets the target value to interpolate to, to the given one
		///
		/// @param targetValue - The value to interpolate to
		constexpr inline auto set_target(T targetValue) noexcept -> void {
			m_target_value = targetValue;
			m_initial_value = m_current_value;
			mLinearTransitionStep
				= (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds);
			m_samples_to_transition = get_samples_to_transition();
		}

		/// @brief Sets the sample rate to use for interpolation
		///
		/// @param sampleRate - The sample rate to use for interpolation
		inline auto set_sample_rate(size_t sampleRate) noexcept -> void {
			m_sample_rate = sampleRate;
			mLinearTransitionStep
				= (m_target_value - m_initial_value)
				  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds);
			m_samples_to_transition = get_samples_to_transition();
			m_current_transition_sample = 0ULL;
		}

		constexpr auto
		operator=(const Interpolator& interpolator) noexcept -> Interpolator& = default;
		constexpr auto operator=(Interpolator&& interpolator) noexcept -> Interpolator& = default;

	  private:
		static constexpr T LOG_TIME_FACTOR = gsl::narrow_cast<T>(5.0);
		static constexpr T EXP_TIME_FACTOR = gsl::narrow_cast<T>(0.693);

		size_t m_sample_rate = DEFAULT_SAMPLE_RATE;
		T m_target_value = DEFAULT_TARGET_VALUE;
		T m_current_value = DEFAULT_INITIAL_VALUE;
		T m_initial_value = DEFAULT_INITIAL_VALUE;
		T m_transition_lengthSeconds = DEFAULT_TRANSITION_LENGTH;
		T mLinearTransitionStep
			= (m_target_value - m_initial_value)
			  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds);
		size_t m_samples_to_transition = get_samples_to_transition();
		size_t m_current_transition_sample = 0;

		/// @brief Gets the interpolated value for the given sample
		///
		/// @param sample - The sample to get the interpolated value for
		///
		/// @return The interpolated value
		inline auto interpolate(size_t sample) noexcept -> T {
			if constexpr(Type == InterpolationType::Linear) {
				return linear_interpolation(sample);
			}
			else if constexpr(Type == InterpolationType::Logarithmic) {
				return log_interpolation(sample);
			}
			else {
				return exp_interpolation(sample);
			}
		}

		inline auto linear_interpolation(size_t sample) noexcept -> T {
			return m_initial_value + (mLinearTransitionStep * gsl::narrow_cast<T>(sample));
		}

		/// @brief Performs "logarithmic" interpolation.
		/// In reality, this is a pseudo logarithmic shape:
		///
		/// y = transition_factor *
		/// [1 + (peak_factor - 1) * (-e^{1 - t / (transtionLength / time_factor) } ) ]
		///
		/// where t = sample / sampleRate
		///
		/// which has a  horizontal asymptote at  ~ the target value and more pronouced slope
		///
		/// @param sample - The sample to get the interpolation value for
		///
		/// @return - The interpolated value
		inline auto log_interpolation(size_t sample) noexcept -> T {
			return (m_initial_value - m_target_value)
					   * math::Exponentials::exp(-gsl::narrow_cast<T>(sample)
												 / (gsl::narrow_cast<T>(m_sample_rate)
													* m_transition_lengthSeconds / LOG_TIME_FACTOR))
				   + m_target_value;
		}

		/// @brief Performs exponential interpolation:
		///
		/// y = e^{t / (transitionLength / time_factor) } - 1
		///
		/// where t = sample / sampleRate
		///
		/// @param sample The sample to get the interpolation value for
		///
		/// @return - The interpolated value
		inline auto exp_interpolation(size_t sample) noexcept -> T {
			return (m_target_value - m_initial_value)
					   * (math::Exponentials::exp(
							  gsl::narrow_cast<T>(sample)
							  / (gsl::narrow_cast<T>(m_sample_rate) * m_transition_lengthSeconds
								 / EXP_TIME_FACTOR))
						  - gsl::narrow_cast<T>(1.0))
				   + m_initial_value;
		}

		/// @brief Gets the number of samples necessary to fully complete the interpolation sequence
		///
		/// @return The number of samples in the interpolation sequence
		inline auto get_samples_to_transition() noexcept -> size_t {
			return gsl::narrow_cast<size_t>(gsl::narrow_cast<T>(m_sample_rate)
											* m_transition_lengthSeconds);
		}
	};
	IGNORE_PADDING_STOP

} // namespace hyperion::math