text:
# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Declare(SSEUnparser);

_toReal := v -> Cond(IsValue(v), Value(TReal, v.v), tcast(TReal, v));

@Value      := @.cond(IsValue);
@TInt       := @.cond(x->IsIntT(x.t));
@TReal      := @.cond(x->IsRealT(x.t));
@TRealInt   := @.cond(x->IsIntT(x.t) or IsRealT(x.t));
@_scalar    := @.cond(x->IsIntT(x.t) or IsRealT(x.t) or IsPtrT(x.t));
@TVect      := @.cond(x->IsVecT(x.t));
@TVectUChar := @.cond(x->IsVecT(x.t) and ObjId(x.t.t)=TUChar);

_isa := self -> self.opts.vector.isa;
_epi_or_px := (self, o) -> When(_isa(self).isFixedPoint or IsOrdT(o.t.t),
    "epi" :: self.ctype_suffixval(o.t, _isa(self)),
    self.ctype_suffix(o.t, _isa(self)));

_epu_or_px := (self, o) -> When(_isa(self).isFixedPoint,
    "epu" :: self.ctype_suffixval(o.t, _isa(self)),
    self.ctype_suffix(o.t, _isa(self)));

_epi := (self, o) -> Concat("epi", self.ctype_suffixval(o.t, _isa(self)));

_epu_to_epi := (s) -> Cond( s{[1..3]} = "epu", "epi" :: s{[4..Length(s)]}, s );

Class(SSEUnparser, CMacroUnparserProg, rec(
    # -----------------------------
    # ISA independent constructs
    # -----------------------------
    nth := (self, o, i, is) >> self.printf("$1[$2]", [o.loc, o.idx]),
    fdiv := (self, o, i, is) >> self.printf("(((double)$1) / ($2))", o.args),
    div  := (self, o, i, is) >> self.printf("(($1) / ($2))", o.args),
    idiv := (self, o, i, is) >> self.printf("(($1) / ($2))", o.args),

    # --------------------------------
    # ISA constructs, general
    # -------------------------------

    # This is a general suffix for intrinsics that is determine from the data type
    ctype_suffix := (self, t, isa) >> Cond(
        t = TVect(T_Int(128), 1),  "epi128",
        t = TVect(T_UInt(128), 1), "epu128",

        t = TVect(T_Real(32), 4) or
        t = TVect(T_Real(32), 2) and isa=SSE_2x32f or
        t = TVect(TReal, 4) and isa=SSE_4x32f or
        t = TVect(TReal, 2) and isa=SSE_2x32f,
        "ps",

        # no way to create __m64 type directly from floats, have to go thru integers
        t = TVect(T_Real(32), 2) and isa=SSE_4x32f or
        t = TVect(TReal, 2) and isa=SSE_4x32f,
        "ps_half",

        t = TVect(TInt, 4) or
        t = TVect(T_Int(32), 4) or
        t = TVect(TReal, 4) and isa.isFixedPoint,
        "epi32",

        t = TVect(T_UInt(32), 4), "epu32",

        t = TVect(T_Real(64), 2) or
        t = TVect(TReal, 2) and isa=SSE_2x64f,
        "pd",

        t = TVect(TInt, 2) or
        t = TVect(T_Int(64), 2) or
        t = TVect(TReal, 2) and isa.isFixedPoint,
        "epi64",

        t = TVect(T_UInt(64), 2), "epu64",

        t = TVect(T_Int(16),  8), "epi16",
        t = TVect(T_UInt(16), 8), "epu16",
        t = TVect(TInt,       8), "epi16",
        t = TVect(TReal,      8), "epi16",

        t = TVect(T_Int(8),  16), "epi8",
        t = TVect(T_UInt(8), 16), "epu8",
        t = TVect(TReal,     16), When(isa.isSigned, "epi8", "epu8"),
        t = TVect(TInt,      16), "epi8",
        t = TVect(TUChar,    16), "epu8",
        ""
    ),

    mul_suffix := (t,isa) -> Cond(
        t = TVect(T_Real(32), 2), "_ps",
        t = TVect(T_Real(32), 4), "_ps",
        t = TVect(T_Real(64), 2), "_pd",
        t = TVect(TReal, 2) and isa = SSE_2x32f, "_ps",
        t = TVect(TReal, 2), "_pd",
        t = TVect(TInt, 2), "_epi64",
        t = TVect(TReal, 4), "_ps",
        t = TVect(TInt, 4), "_epi32",
        t = TVect(T_Int(32), 4), "lo_epi32",
        t = TVect(TReal, 8), "lo_epi16",
        t = TVect(TReal, 16), Error("16-way multiplication is not supported"),
        t = TVect(TUChar, 16), Error("16-way multiplication is not supported"),
        ""
    ),

    # This is a general suffix for intrinsics that is determine from the data type
    ctype_suffixval := (t, isa) -> Cond(
	t = TVect(TReal, 2), "64",
	t = TVect(TInt, 4), "32",
	t = TVect(TReal, 4), "32",
	t = TVect(T_Int(32), 4), "32",
    t = TVect(T_UInt(32), 4), "32",
	t = TVect(TReal, 8), "16",
	t = TVect(TInt, 8), "16",
	t = TVect(T_Int(16), 8), "16",
    t = TVect(T_UInt(16), 8), "16",
	t = TVect(TReal, 16), "8",
	t = TVect(TInt, 16), "8",
	t = TVect(T_Int(8), 16), "8",
    t = TVect(T_UInt(8), 16), "8",
	t = TVect(TUChar, 16), "8",
	t = TVect(T_Real(32), 4), "32",
	t = TVect(T_Real(64), 2), "64",
	""
    ),

    # This is the type used for declarations of vector variables
    ctype := (self, t, isa) >> Cond(
        # NOTE: used for unaligned vector pointers,for single prec, it should be "float"
	t in [TReal, TVect(TReal, 1)],
            Cond(isa = SSE_2x64f, "double",
		 isa = SSE_2x64i, "__int64",
		 isa = SSE_4x32f, "float",
		 isa = SSE_2x32f, "float",
		 isa = SSE_4x32i, "__int32",
		 isa = SSE_8x16i, "short",
		 isa = SSE_16x8i, Cond(isa.isSigned, "char", "unsigned char"),
		 isa.ctype),

	t = TVect(TReal, 2),
            Cond(isa = SSE_2x64f, "__m128d",
		 isa = SSE_2x64i, "__m128i",
		 isa = SSE_4x32f, "__m64",
		 isa = SSE_8x16i, "__int32",
		 isa = SSE_16x8i, "__int16",
		 isa = SSE_2x32f, "__m64"),

	t = TVect(TReal, 4),
            Cond(isa = SSE_4x32f, "__m128",
		 isa = SSE_2x32f, "__m128",
		 isa = SSE_4x32i, "__m128i"),

        t = TVect(TInt,    2), "__m128i",
	t = TVect(TInt,    4), "__m128i",
	t = TVect(TInt,    8), "__m128i",
	t = TVect(TReal,   8), "__m128i",
	t = TVect(TInt,   16), "__m128i",
	t = TVect(TUChar, 16), "__m128i",
	t = TVect(TReal,  16), "__m128i",

	t = TInt,
            Cond(isa = SSE_2x64i, "__int64",
		 isa = SSE_4x32i, "__int32",
		 isa = SSE_8x16i, "short",
		 isa = SSE_16x8i, "char",
		 "int"),

	t = TVect(T_Int(128), 1), "__m128i",
	t = TVect(T_Int(64),  2), "__m128i",
	t = TVect(T_Int(32),  4), "__m128i",
	t = TVect(T_Int(16),  8), "__m128i",
	t = TVect(T_Int(8),  16), "__m128i",

	t = TVect(T_UInt(128), 1), "__m128i",
	t = TVect(T_UInt(64),  2), "__m128i",
	t = TVect(T_UInt(32),  4), "__m128i",
	t = TVect(T_UInt(16),  8), "__m128i",
	t = TVect(T_UInt(8),  16), "__m128i",

        t = TVect(T_Real(32), 2), "__m64",
	t = TVect(T_Real(32), 4), "__m128",
	t = TVect(T_Real(64), 2), "__m128d",
	Error(self,".ctype doesn't know type ",t)
    ),

    cvalue_suffix  := (self, t)  >> let( isa := _isa(self), Cond(
        (t = TReal and isa in [SSE_2x32f, SSE_4x32f]) or t = T_Real(32), "f",
        (t = TReal) or t = T_Real(64), "",
        Error(self,".cvalue_suffix doesn't know type ",t)
    )),

    vhex := (self, o, i, is) >> Print("_mm_set_", _epi(self, o), "(", self.infix(Reversed(o.p), ", "), ")"),

    Value := (self, o, i, is) >> let(zero := "0" :: self.cvalue_suffix(TReal), Cond(
        o.t = TString, Print(o.v),

        o.t = TReal or ObjId(o.t)=T_Real, let(v := When(IsCyc(o.v), ReComplex(Complex(o.v)), Double(o.v)),
            When(v<0, Print("(", v, self.cvalue_suffix(o.t), ")"), Print(v, self.cvalue_suffix(o.t)))),

        #IsComplexT(o.t),
	#    Print("COMPLEX(", ReComplex(Complex(o.v)), self.cvalue_suffix(o.t.realType()), ", ",
	#        ImComplex(Complex(o.v)), self.cvalue_suffix(o.t.realType()), ")"),

        IsIntT(o.t) or IsUIntT(o.t),
            When(o.v < 0, Print("(", o.v, ")"), Print(o.v)),

        ObjId(o.t) = TVect and _isa(self) = SSE_2x32f,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print(          "{", zero, ", ", zero, ", ", self.infix((o.v), ", "), "}"),
		 Print("_mm_set_ps(", zero, ", ", zero, ", ", self.infix(Reversed(o.v), ", "), ")")),

        ObjId(o.t) = TVect and Length(Set(o.v)) = 1,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print("{", self.infix(Replicate(o.t.size, o.v[1]), ", "), "}"),
		 Print("_mm_set1_", _epi_or_px(self, o), "(", self(o.v[1], i, is), ")")),

        ObjId(o.t) = TVect,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print(                                 "{", self.infix((o.v), ", "), "}"),
		 Print("_mm_set_", _epi_or_px(self, o), "(", self.infix(Reversed(o.v), ", "), ")")),

        IsArray(o.t),
            Print("{", self.infix(o.v, ", "), "}"),

        ObjId(o.t) = TSym,
            Print("(", self.declare(o.t, [], 0, 0), ") ", o.v),

        o.t = TBool, Print(When(o.v = true, "1", "0")),

	Inherited(o, i, is)
    )),

    vpack := (self, o, i, is) >> let(
        sfx := _epi_or_px(self, o),
        Print("_mm_set_", sfx, "(", self.infix(Reversed(o.args), ", "), ")")),

    vdup := (self, o, i, is) >> let(
	sfx := _epi_or_px(self, o),
        CondPat(o,
            [vdup, nth, @.cond(x->x.t=TInt and x.v=2)], self.printf("_mm_loaddup_$1(&($2))", [sfx, o.args[1]]),
            [vdup, @, @TInt], self.printf("_mm_set1_$1($2)", [sfx, o.args[1]]))),

    # --------------------------------
    # Declarations
    _declTVect := (self, t, vars, i, is) >> let(ctype := self.ctype(t, _isa(self)), Print(ctype, " ", self.infix(vars, ", ", i+is))),
    _unparseTVect := (self, t, i, is) >> let(ctype := self.ctype(t, _isa(self)), Print(ctype)),

    TVect := arg >> When(Length(arg)=5, arg[1]._declTVect(arg[2], arg[3], arg[4], arg[5]),
	                                arg[1]._unparseTVect(arg[2], arg[3], arg[4])),
    TReal := ~.TVect,
    TInt  := (self, t, vars, i, is) >> Print("int ", self.infix(vars, ", ", i+is)),
    TBool := (self, t, vars, i, is) >> Print("int ", self.infix(vars, ", ", i+is)),

    # --------------------------------
    # Arithmetic
    #
    mul := (self, o, i, is) >> let(n := Length(o.args), Cond(
        not IsVecT(o.t),
            Print("(",self.pinfix(o.args, ")*("),")"),
	n > 2 and n mod 2 <> 0,
            self(mul(o.args[1], ApplyFunc(mul, Drop(o.args, 1))), i, is),
        n > 2,
            self(mul(ApplyFunc(mul, o.args{[1..n/2]}), ApplyFunc(mul, o.args{[n/2+1..n]})), i, is),
        CondPat(o,
	    [mul, @TReal, @TVect], Cond(_isa(self) = SSE_2x32f,
                self(mul(vdup(o.args[1], 4), o.args[2]), i, is), # NOTE: HACK for SSE_2x32f
                self(mul(vdup(o.args[1], o.t.size), o.args[2]), i, is)),
	    [mul, @TVect, @TReal],  self(mul(o.args[1], vdup(o.args[2],o.t.size)), i, is),
            # NOTE: This hack is probably no longer necessary (was used for PRDFTs)
	    [mul, @(1, cond, e -> e.t=TInt), @TVect],
	        self(mul(cond(o.args[1].args[1],
			      vdup(o.t.t.value(o.args[1].args[2]), o.t.size),
			      vdup(o.t.t.value(o.args[1].args[3]), o.t.size)), o.args[2]), i, is),
	    [mul, @TInt, @TVect],  self(mul(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
	    [mul, @TVect, @TInt],  self(mul(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
	    [mul, @TVect, @TVect], self.printf("_mm_mul$1($2, $3)", [self.mul_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
	    Error("Don't know how to unparse <o>. Unrecognized type combination")
    ))),

    fpmul := (self, o, i, is) >> let(isa := _isa(self), CondPat(o,
        # preparing for SSSE3 _mm_mulhrs_epi16 (__m128i a, __m128i b)
        # self.printf("_mm_mulhrs_epi16($1, $2)", [o.args[2], o.args[3].t.value(List(o.args[3].v, i->bin_shl(i,1)))]),
        [fpmul, @, @TVect, @], self.printf("$1($2($3, $4), $5)",
	    [isa.vlshift, isa.vmul, o.args[2], o.args[3], isa.bits-o.args[1]]),

        [fpmul, @, @, @],      self.printf("$1($2(_mm_set1_$3($4), $5), $6)",
	    [isa.vlshift, isa.vmul, self.ctype_suffix(o.t, _isa(self)), o.args[2], o.args[3], isa.bits-o.args[1]]))),

    add := (self, o, i, is) >> let(n := Length(o.args), Cond(
	not IsVecT(o.t),
            self.pinfix(o.args, " + "),
        n > 2 and n mod 2 <> 0,
            self(add(o.args[1], ApplyFunc(add, Drop(o.args, 1))), i, is),
        n > 2,
            self(add(ApplyFunc(add, o.args{[1..n/2]}), ApplyFunc(add, o.args{[n/2+1..n]})), i, is),
        let(isa := _isa(self), # ugly, backward compatibility, use <adds> instead
	    saturated := When(IsBound(isa.isFloat) and IsBound(isa.saturatedArithmetic) and not isa.isFloat and isa.saturatedArithmetic, "s", ""),
	    _sfx      := self.ctype_suffix(o.t, isa),
	    sfx       := Cond( saturated="", _epu_to_epi(_sfx), _sfx),
	    CondPat(o,
		[add, @TVect,   @TVect], self.printf("_mm_add$1_$2($3, $4)", [saturated, sfx, o.args[1], o.args[2]]),
		Error("Don't know how to unparse <o>. Unrecognized type combination"))))),

    adds := (self, o, i, is) >> CondPat(o,
		[adds, @TVect, @TVect, ...],
		    Cond( Length(o.args)>2,
		        self(adds(o.args[1], brackets(ApplyFunc(adds, Drop(o.args, 1)))), i, is),
		        self.printf("_mm_adds_$1($2, $3)", [self.ctype_suffix(o.t, rec()), o.args[1], o.args[2]])),
		Inherited(o, i, is)),

    _sub := (self, t, a, i, is) >> let(
	isa := _isa(self),
	sfx := _epu_to_epi(self.ctype_suffix(t, isa)),
	saturated := When(IsBound(isa.isFloat) and IsBound(isa.saturatedArithmetic) and not isa.isFloat and isa.saturatedArithmetic, "s", ""),
	CondPat(a,
            [ListClass, @TVect,   @TVect], self.printf("_mm_sub$1_$2($3, $4)", [saturated, sfx, a[1], a[2]]),
            [ListClass, @, @],             self.printf("($1 - ($2))", a),
            Error("Don't know how to unparse subtraction of a[1] and a[2]. Unrecognized type combination"))),

    sub := (self, o, i, is) >> self._sub(o.t, o.args, i, is),

    neg := (self, o, i, is) >> CondPat(o,
        [@, @TVect], self._sub(o.t, [o.t.zero(), o.args[1]], i, is),
        self.printf("(-$1)", o.args)),

    stickyNeg := ~.neg,

    sqrt  := (self, o, i, is) >> Cond( IsVecT(o.t),
        Checked( IsRealT(o.t.t), self.printf("_mm_sqrt_$1($2)", [self.ctype_suffix(o.t, _isa(self)), o.args[1]])),
        Inherited(o, i, is)),

    rsqrt := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx="ps", self.printf("_mm_rsqrt_ps($1)", [o.args[1]]))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    log := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            Length(o.args)>1 and (o.args[2]=2 or o.args[2]=o.t.value(2)),
                self.printf("_mm_log2_$1($2)", [sfx, o.args[1]]),
            Length(o.args)>1 and (o.args[2]=10 or o.args[2]=o.t.value(10)),
                self.printf("_mm_log10_$1($2)", [sfx, o.args[1]]),
            Length(o.args)=1 or o.args[2]=d_exp(1) or o.args[2]=o.t.value(d_exp(1)),
                self.printf("_mm_log_$1($2)", [sfx, o.args[1]]),
            self.printf("_mm_div_$1(_mm_log_$1($2), _mm_log_$1($3))", [sfx, o.args[1]])))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    exp := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], self.printf("_mm_exp_$1($2)", [sfx, o.args[1]]))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    pow := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            o.args[1]=2 or o.args[1]=o.t.value(2),
                self.printf("_mm_exp2_$1($2)", [sfx, o.args[2]]),
            o.args[1]=d_exp(1) or o.args[1]=o.t.value(d_exp(1)),
                self.printf("_mm_exp_$1($2)", [sfx, o.args[2]]),
            self.printf("_mm_pow_$1($2, $3)", [sfx, o.args[1], o.args[2]])))),
        Inherited(o, i, is)),

    imod  := (self, o, i, is) >> Cond( IsIntT(o.t.base_t()) and Is2Power(o.args[2]),
        # in two's complement arithmetics this will work for both positive and negative o.args[1]
        self(bin_and(o.args[1], o.args[2]-1), i, is),
        self.printf("(($1) % ($2))", o.args)),

    # --------------------------------
    # logic
    #
    arith_shl := (self, o, i, is) >> self.prefix(_isa(self).vlshift, o.args),
    arith_shr := (self, o, i, is) >> CondPat( o,
        [arith_shr, @.cond(x->x.t=TVect(T_Int(32), 4)), @],
            self.prefix("_mm_srai_epi32", o.args),
        [arith_shr, @.cond(x->x.t=TVect(T_Int(16), 8)), @],
            self.prefix("_mm_srai_epi16", o.args),
        [arith_shr, @TVect, @],
            self.prefix(_isa(self).vrshift, o.args),
        Inherited(o, i, is)),

    bin_xor := (self, o, i, is) >> CondPat(o,
                [bin_xor, @TVect, @TVect], self.prefix("_mm_xor_si128", o.args),
                Inherited(o, i, is)),
    bin_and := (self, o, i, is) >> CondPat(o,
                [bin_and, @TVect, @TVect], self.prefix("_mm_and_si128", o.args),
                Inherited(o, i, is)),

    bin_andnot := (self, o, i, is) >> self.prefix("_mm_andnot_si128", o.args),

    bin_or := (self, o, i, is) >> CondPat(o,
        [bin_or, @TVect, @TVect], let(sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( not (sfx in ["ps", "pd", "ps_half"]), #was: _isa(self).isFixedPoint,
		self.printf("_mm_or_si128($1, $2)", o.args),
		self.printf("_mm_castsi128_$3(_mm_or_si128(_mm_cast$3_si128($1), _mm_cast$3_si128($2)))",
		            o.args :: [sfx]))),
        [bin_or, @TReal, @TReal], self.printf("(($1) | ($2))", o.args),

	Inherited(o, i, is)),

    min := (self, o, i, is) >> CondPat(o,
        [min, @TVect, @TVect], self.prefix("_mm_min_" :: self.ctype_suffix(o.t, _isa(self)), o.args),
            Inherited(o, i, is)),

    max := (self, o, i, is) >> let(n := Length(o.args), When(
    	IsVecT(o.t) and n > 2, self.printf("_mm_max_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], ApplyFunc(max, Drop(o.args, 1))]),
        CondPat(o,
            [max, @TVect, @TVect], self.prefix("_mm_max_" :: self.ctype_suffix(o.t, _isa(self)), o.args),
                Inherited(o, i, is)))),

    abs := (self, o, i, is) >> CondPat(o,
        [abs, @TVect], let( sfx := self.ctype_suffix(o.t, _isa(self)),
                            Cond( sfx = "ps", self.printf("_mm_castsi128_ps(_mm_and_si128(_mm_castps_si128($1), _mm_set_epi32(0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF)))", o.args),
                                  sfx = "pd", self.printf("_mm_castsi128_pd(_mm_and_si128(_mm_castpd_si128($1), _mm_set_epi32(0x7FFFFFFF, 0xFFFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF)))", o.args),
                                  Error("not implemented"))),
            Inherited(o, i, is)),

    bin_shl := (self, o, i, is) >> CondPat(o,
        [bin_shl, @TVect, @TInt], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( _isa(self).isFixedPoint,                            # legacy
		     self.printf("_mm_slli_si128($1, $2)", o.args),   # legacy
		  sfx in ["epi16", "epi32", "epi64", "epu16", "epu32", "epu64"],
		      self.printf("_mm_slli_$3($1, $2)", o.args :: [_epu_to_epi(sfx)]),
		  sfx in ["epi8", "epu8"],
		      Error("bin_shl is undefined for epi8 and epu8"),
		  sfx in ["epi128", "epu128"], # shift with byte granularity
		      self.printf("_mm_slli_si128($1, $2)", [o.args[1], idiv(o.args[2], 8)] ),
		  # else, shift whole register with shift argument specified in bytes (legacy, fix using epi128 in ISAs first)
		  self.printf("_mm_castsi128_$3(_mm_slli_si128(_mm_cast$3_si128($1), $2))", o.args :: [sfx]))),
        [bin_shl, @TReal, @TInt], self.printf("(($1) << ($2))", o.args),
        [bin_shl, @TInt, @TInt], self.printf("(($1) << ($2))", o.args),
        [bin_shl, @, @], self.prefix("_mm_slli_" :: self.ctype_suffix(o.t, _isa(self)), o.args)),

    bin_shr := (self, o, i, is) >> CondPat(o,
        [bin_shr, @TVect, @TInt], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( _isa(self).isFixedPoint,                            # legacy
		      self.printf("_mm_srli_si128($1, $2)", o.args),  # legacy
		  sfx in ["epi16", "epi32", "epi64", "epu16", "epu32", "epu64"],
		      self.printf("_mm_srli_$3($1, $2)", o.args :: [_epu_to_epi(sfx)]),
		  sfx in ["epi8", "epu8"],
		      Error("bin_shr is undefined for epi8 and epu8"),
		  sfx in ["epi128", "epu128"], # shift with byte granularity
		      self.printf("_mm_srli_si128($1, $2)", [o.args[1], idiv(o.args[2], 8)] ),
		  # else, shift whole register with shift argument specified in bytes (legacy, fix using epi128 in ISAs first)
		  self.printf("_mm_castsi128_$3(_mm_srli_si128(_mm_cast$3_si128($1), $2))", o.args :: [sfx]))),
	# default
	[bin_shr, @TReal, @TInt], self.printf("(($1) >> ($2))", o.args),
	[bin_shr, @TInt, @TInt], self.printf("(($1) >> ($2))", o.args),
	# what's this?
	[bin_shr, @, @], self.prefix("_mm_srli_" :: self.ctype_suffix(o.t, _isa(self)), o.args)),

    # vector shifts
    vec_shr := (self, o, i, is) >> let(
        isa := _isa(self),
        sfx := self.ctype_suffix(o.t, isa),
        # making sure this is SSE data type
        t   := Checked(IsVecT(o.t) and sfx<>"", o.t),
        a   := o.args[1],
        s   := o.args[2] * 16 / t.size,
        # may need typecasts to please compiler
        Cond( self.ctype(t, isa)="__m128i",
            self.printf("_mm_srli_si128($1, $2)", [a, s] ),
            self.printf("_mm_castsi128_$3(_mm_srli_si128(_mm_cast$3_si128($1), $2))", [a, s, sfx])
        )),

    vec_shl := (self, o, i, is) >> let(
        isa := _isa(self),
        sfx := self.ctype_suffix(o.t, isa),
        # making sure this is SSE data type
        t   := Checked(IsVecT(o.t) and sfx<>"", o.t),
        a   := o.args[1],
        s   := o.args[2] * 16 / t.size,
        # may need typecasts to please compiler
        Cond( self.ctype(t, isa)="__m128i",
            self.printf("_mm_slli_si128($1, $2)", [a, s] ),
            self.printf("_mm_castsi128_$3(_mm_slli_si128(_mm_cast$3_si128($1), $2))", [a, s, sfx])
        )),

    # --------------------------------
    # comparison
    #
    eq := (self, o, i, is) >> let( ctype := self.ctype_suffix(o.args[1].t, _isa(self)),
        sfx := _epu_to_epi(ctype),
        Cond(IsVecT(o.t), self.prefix("_mm_cmpeq_" :: sfx, o.args),
            Inherited(o, i, is))),

    lt := (self, o, i, is) >> Cond(IsVecT(o.t),
        self.prefix("_mm_cmplt_" :: self.ctype_suffix(o.args[1].t, _isa(self)), o.args),
        Inherited(o, i, is)),

    gt := (self, o, i, is) >> Cond(ObjId(o.t)=TVect,
        self.prefix("_mm_cmpgt_" :: self.ctype_suffix(o.args[1].t, _isa(self)), o.args),
        Inherited(o, i, is)),

    mask_eq := ~.eq,
    mask_lt := ~.lt,
    mask_gt := ~.gt,

    vparam := (self, o, i, is) >> iclshuffle(o.p),

    # --------------------------------
    # ISA specific : SSE_2x64f
    #
    vunpacklo_2x64f := (self, o, i, is) >> self.prefix("_mm_unpacklo_pd", o.args),
    vunpackhi_2x64f := (self, o, i, is) >> self.prefix("_mm_unpackhi_pd", o.args),
    vshuffle_2x64f  := (self, o, i, is) >> self.prefix("_mm_shuffle_pd", o.args),
    vushuffle_2x64f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),

    vload1sd_2x64f := (self, o, i, is) >> self.prefix("_mm_load_sd", o.args),
    vload_1l_2x64f := (self, o, i, is) >> self.prefix("_mm_loadl_pd", o.args),
    vload_1h_2x64f := (self, o, i, is) >> self.prefix("_mm_loadh_pd", o.args),
    vloadu_2x64f   := (self, o, i, is) >> self.prefix("_mm_loadu_pd", o.args),

    vstore_1l_2x64f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pd", o.args), ";\n"),
    vstore_1h_2x64f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeh_pd", o.args), ";\n"),
    vstoreu_2x64f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_pd", o.args), ";\n"),

    addsub_2x64f := (self, o, i, is) >> Checked(Length(o.args) = 2,
        CondPat(o,
           [addsub_2x64f, @TReal, @TVect], self(addsub_2x64f(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [addsub_2x64f, @TVect, @TReal], self(addsub_2x64f(o.args[1], vdup(o.args[2], o.t.size)), i, is),
           [addsub_2x64f, @TInt,  @TVect], self(addsub_2x64f(vdup(_toReal(o.args[1]), o.t.size), o.args[2]), i, is),
           [addsub_2x64f, @TVect, @TInt],  self(addsub_2x64f(o.args[1], vdup(_toReal(o.args[2]), o.t.size)), i, is),
           [addsub_2x64f, @TVect, @TVect], self.printf("_mm_addsub_pd($1, $2)", o.args),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
    )),

    hadd_2x64f := (self, o, i, is) >> self.printf("_mm_hadd_pd($1, $2)", [o.args[1], o.args[2]]),

    chslo_2x64f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_pd(_mm_xor_si128(_mm_castpd_si128($1), _mm_set_epi32(0, 0, 0x80000000, 0)))", o.args),
    chshi_2x64f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_pd(_mm_xor_si128(_mm_castpd_si128($1), _mm_set_epi32(0x80000000, 0, 0, 0)))", o.args),
    chshi_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128($1), _mm_set_epi32(0x80000000, 0, 0x80000000, 0)))", o.args),
    chslo_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128($1), _mm_set_epi32(0, 0x80000000, 0, 0x80000000)))", o.args),

    vcvt_64f32f := (self, o, i, is) >> self.prefix("_mm_cvtps_pd", o.args),

    cmpge_2x64f := (self, o, i, is) >> self.prefix("_mm_cmpge_pd", o.args),

    cmple_2x64f := (self, o, i, is) >> self.prefix("_mm_cmple_pd", o.args),
    cmpeq_2x64f := (self, o, i, is) >> self.prefix("_mm_cmpeq_pd", o.args),

    # --------------------------------
    # ISA specific : SSE_2x32f
    #
    vunpacklo_2x32f := (self, o, i, is) >> self.prefix("_mm_unpacklo_ps", o.args),
    vunpackhi_2x32f := (self, o, i, is) >> self.prefix("_mm_unpackhi_ps", o.args),
    vshuffle_2x32f  := (self, o, i, is) >> self.prefix("_mm_shuffle_ps", o.args),
    vushuffle_2x32f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    vload_2x32f     := (self, o, i, is) >> self.prefix("_mm_loadl_pi", o.args),
    vstore_2x32f    := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pi", o.args), ";\n"),
    vstoreu_2x32f   := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_storel_epi64($1, _mm_castps_si128($2));\n", o.args)),
    vloadu_2x32f    := (self, o, i, is) >> self.printf("_mm_castsi128_ps(_mm_loadl_epi64($1))", o.args),

    # --------------------------------
    # ISA specific : SSE_4x32f
    #
    prefix_cast := (self, prefix, t, o) >> Cond( self.ctype(t, _isa(self)) = self.ctype(o.t, _isa(self)), self.prefix(prefix, o.args),
                                             self(tcast(o.t, ApplyFunc(ObjId(o), List(o.args, a -> Cond(IsExp(a), tcast(t, a), a)))), 0, 1)),

    vunpacklo_4x32f := (self, o, i, is) >> self.prefix_cast("_mm_unpacklo_ps", TVect(T_Real(32), 4), o),
    vunpackhi_4x32f := (self, o, i, is) >> self.prefix_cast("_mm_unpackhi_ps", TVect(T_Real(32), 4), o),
    vshuffle_4x32f  := (self, o, i, is) >> self.prefix_cast("_mm_shuffle_ps", TVect(T_Real(32), 4), o),
    vushuffle_4x32f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    hadd_4x32f  := (self, o, i, is) >> self.printf("_mm_hadd_ps($1, $2)", [o.args[1], o.args[2]]),
    vldup_4x32f := (self, o, i, is) >> self.prefix("_mm_moveldup_ps", o.args),
    vhdup_4x32f := (self, o, i, is) >> self.prefix("_mm_movehdup_ps", o.args),

    vinsert_4x32f  := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_insert_epi32(_mm_castps_si128($1), $2, $3))", [o.args[1], o.args[2], o.args[3].p-1]),
    vextract_4x32f := (self, o, i, is) >> Print(Blanks(i),
	self.printf("$1 = _mm_extract_ps($2, $3)", [deref(o.args[1]), o.args[2], o.args[3]-1]), ";\n"),

    vload1_4x32f   := (self, o, i, is) >> self.prefix("_mm_load_ss", o.args),
    vload_2l_4x32f := (self, o, i, is) >> self.prefix("_mm_loadl_pi", o.args),
    vload_2h_4x32f := (self, o, i, is) >> self.prefix("_mm_loadh_pi", o.args),
    vloadu_4x32f   := (self, o, i, is) >> self.printf("_mm_loadu_ps($1)", o.args),
    vloadu2_4x32f  := (self, o, i, is) >> self.printf("_mm_castsi128_ps(_mm_loadl_epi64($1))", o.args),

    vstore1_4x32f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_store_ss",  o.args), ";\n"),
    vstore_2l_4x32f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pi", o.args), ";\n"),
    vstore_2h_4x32f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeh_pi", o.args), ";\n"),
    vstoreu_4x32f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_ps", o.args), ";\n"),
    vstoreu2_4x32f  := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_storel_epi64($1, _mm_castps_si128($2));\n",
        o.args)),

    vstoremsk_4x32f := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128(_mm_castps_si128($2), _mm_set_epi32($3, $4, $5, $6), $1);\n",
            [o.args[1], o.args[2]] :: List(Reversed(o.args[3].v), e->e.v))),

    alignr_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128($1), _mm_castps_si128($2), $3))", [o.args[1], o.args[2], o.args[3].p]),

    # --------------------------------
    # ISA specific : SSE_8x16i
    #
    vzero_8x16i := (self, o, i, is) >> Print("_mm_setzero_si128()"),

    vunpacklo_8x16i  := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi16", o.args),
    vunpackhi_8x16i  := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi16", o.args),
    vunpacklo2_8x16i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi2_8x16i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vunpacklo4_8x16i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi4_8x16i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),

    vpacks_8x16i     := (self, o, i, is) >> self.prefix("_mm_packs_epi16",    o.args),
    vpackus_8x16i    := (self, o, i, is) >> self.prefix("_mm_packus_epi16",   o.args),

    vshuffle2_8x16i := (self, o, i, is) >> self.printf(
	"_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps($1), _mm_castsi128_ps($2), $3))", o.args),

    vshuffle4_8x16i := (self, o, i, is) >> self.printf(
	"_mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd($1), _mm_castsi128_pd($2), $3))", o.args),

    vload1_8x16i := (self, o, i, is) >> self.printf("_mm_insert_epi16($1, $2, $3)", o.args),
    vload2_8x16i := (self, o, i, is) >> self.prefix("_mm_cvtsi32_si128", o.args),
    vload4_8x16i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vloadu_8x16i := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),

    vextract1_8x16i := (self, o, i, is) >> self.prefix("_mm_extract_epi16", o.args),
    vextract2_8x16i := (self, o, i, is) >> self.prefix("_mm_cvtsi128_si32", o.args),
    vstoreu_8x16i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),
    vstore4_8x16i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_epi64", o.args), ";\n"),
    vstoremsk_8x16i := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128($2, _mm_set_epi16($3), $1);\n",
        [o.args[1], o.args[2], () -> PrintCS(Reversed(o.args[3]))])),

    vushuffle2_8x16i  := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    vushufflelo_8x16i := (self, o, i, is) >> self.prefix("_mm_shufflelo_epi16", o.args),
    vushufflehi_8x16i := (self, o, i, is) >> self.prefix("_mm_shufflehi_epi16", o.args),

    interleavedmask_8x16i := (self, o, i, is) >> self.printf(
	"_mm_movemask_epi8(_mm_unpacklo_epi8(_mm_packs_epi16($1, _mm_setzero_si128()), _mm_packs_epi16($2, _mm_setzero_si128())))",
	o.args),

    alignr_8x16i := (self, o, i, is) >> self.printf("_mm_alignr_epi8($1, $2, $3)", [o.args[1], o.args[2], o.args[3].p]),

    # FF: NOTE: couldnt figure out how to use the general case with type propagation etc...
    cmplt_8x16i := (self, o, i, is) >> self.printf("_mm_cmplt_epi16($1, $2)", o.args),

    # SSSE3 8x16i instructions
    chs_8x16i := (self, o, i, is) >> self.prefix("_mm_sign_epi16", o.args),
    vushuffle_8x16i := (self, o, i, is) >> self.prefix("_mm_shuffle_epi8", o.args),

    # --------------------------------
    # ISA specific : SSE_16x8i
    #
    vloadu_16x8i  := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),
    vstoreu_16x8i := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),

    vunpacklo_16x8i  := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi8", o.args),
    vunpackhi_16x8i  := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi8", o.args),
    vunpacklo2_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi16", o.args),
    vunpackhi2_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi16", o.args),
    vunpacklo4_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi4_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vunpacklo8_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi8_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),

    vushufflelo2_16x8i := (self, o, i, is) >> self.prefix("_mm_shufflelo_epi16", o.args),
    vushufflehi2_16x8i := (self, o, i, is) >> self.prefix("_mm_shufflehi_epi16", o.args),
    vushuffle4_16x8i   := (self, o, i, is) >> self.prefix("_mm_shuffle_epi32", o.args),

    interleavedmasklo_16x8i := (self, o, i, is) >> Print("_mm_movemask_epi8(_mm_unpacklo_epi8(",o.args[1],",",o.args[2],"))"),
    interleavedmaskhi_16x8i := (self, o, i, is) >> Print("_mm_movemask_epi8(_mm_unpackhi_epi8(",o.args[1],",",o.args[2],"))"),
    average_16x8i           := (self, o, i, is) >> Print("_mm_avg_epu8(",o.args[1],",",o.args[2],")"),
    vmovemask_16x8i         := (self, o, i, is) >> self.prefix("_mm_movemask_epi8", o.args),

    # XXX NOTE XXXX
    # Also fix other vstoremsk's. The problem here is that after latest changes to Spiral
    # the last argument (list of strings) in vstoremsg gets wrapped into V, and strings too
    # this is super stupid+ugly
    vstoremsk_16x8i := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_maskmoveu_si128($2, _mm_set_epi8($3), $1);\n",
        [o.args[1], o.args[2], () -> PrintCS(Reversed(List(_unwrapV(o.args[3]), _unwrapV)))])),

    addsub_4x32f := (self, o, i, is) >> Checked(Length(o.args) = 2,
        CondPat(o,
           [addsub_4x32f, @TReal, @TVect], self(addsub_4x32f(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [addsub_4x32f, @TVect, @TReal], self(addsub_4x32f(o.args[1], vdup(o.args[2],o.t.size)), i, is),
           [addsub_4x32f, @TInt,  @TVect], self(addsub_4x32f(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
           [addsub_4x32f, @TVect, @TInt],  self(addsub_4x32f(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
           [addsub_4x32f, @TVect, @TVect], self.printf("_mm_addsub_ps($1, $2)", [o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
    )),

    hadd_4x32f   := (self, o, i, is) >> self.printf("_mm_hadd_ps($1, $2)", [o.args[1], o.args[2]]),

    vloadu_16x8i := (self, o, i, is) >>  self.prefix("_mm_loadu_si128", o.args),

    # --------------------------------
    # ISA specific : SSE_4x32i
    #
    vunpacklo_4x32i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi_4x32i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vpacks_4x32i    := (self, o, i, is) >> self.prefix("_mm_packs_epi32", o.args),
    # 32 bit integer shuffles are *not* the same as 32 bit float, but similar to 16 bit integer shuffles
    vushuffle_4x32i := (self, o, i, is) >> self.prefix("_mm_shuffle_epi32", o.args),
    vshuffle_4x32i  := (self, o, i, is) >> self.printf(
	"_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps($1), _mm_castsi128_ps($2), $3))", o.args),

    # subvector unparsing not yet done...
    vload1_4x32i := (self, o, i, is) >> self.prefix("_mm_cvtsi32_si128", o.args), #svpcprint guy
    vload2_4x32i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vload2_4x32i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vloadu_4x32i := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),

    vextract_4x32i  := (self, o, i, is) >> self.prefix("_mm_cvtsi128_si32", o.args),
    vstoreu_4x32i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),
    vstore2_4x32i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_epi64", o.args), ";\n"),
    vstoremsk_4x32i := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128($2, _mm_set_epi16($3), $1);\n", [o.args[1], o.args[2], ()->PrintCS(Reversed(o.args[3]))])),
        # complicated, buggy

    interleavedmask_4x32i := (self, o, i, is) >> self.printf(
	"_mm_movemask_epi8(_mm_packs_epi16(_mm_unpacklo_epi16(_mm_packs_epi16($1, $3), _mm_packs_epi16($2, $3)), $3))",
	o.args :: ["_mm_setzero_si128()"]),

    vcvt_4x32_i2f  := (self, o, i, is) >> self.prefix("_mm_cvtepi32_ps", o.args),
    vcvt_4x32_f2i  := (self, o, i, is) >> self.prefix("_mm_cvtps_epi32", o.args),
    vcvtt_4x32_f2i := (self, o, i, is) >> self.prefix("_mm_cvttps_epi32", o.args),

    testz_4x32i := (self, o, i, is) >> self.prefix("_mm_testz_si128", o.args),
    testc_4x32i := (self, o, i, is) >> self.prefix("_mm_testc_si128", o.args),
    testnzc_4x32i := (self, o, i, is) >> self.prefix("_mm_testnzc_si128", o.args),

    # --------------------------------
    # ISA specific : SSE_2x64i
    #
    vunpacklo_2x64i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi_2x64i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),
    vshuffle_2x64i  := (self, o, i, is) >> self.printf(
	"_mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd($1), _mm_castsi128_pd($2), $3))", o.args),
    vushuffle_2x64i := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),

    # --------------------------------
    # tcast __m128 <-> __m128i

    tcast := (self, o, i, is) >> let(
        isa := _isa(self),
        i128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128i")),
        f128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128" )),
        d128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128d" )),
        CondPat(o,
            [tcast, i128, f128], self.prefix("_mm_castps_si128", [o.args[2]]),
            [tcast, i128, d128], self.prefix("_mm_castpd_si128", [o.args[2]]),
            [tcast, f128, i128], self.prefix("_mm_castsi128_ps", [o.args[2]]),
            [tcast, i128, i128], self(o.args[2], i, is),
            [tcast, f128, f128], self(o.args[2], i, is),
            Inherited(o, i, is))),

    tcvt := (self, o, i, is) >> self.printf("(($1)($2))", [o.args[1], o.args[2]]),

    #NOTE: finish this, it should look at TVect.size and instruction set for figuring out exactly what to do
    vcastizxlo := (self, o, i, is) >> self(vunpacklo_16x8i(o.args[1], o.t.zero()), i, is),
    vcastizxhi := (self, o, i, is) >> self(vunpackhi_16x8i(o.args[1], o.t.zero()), i, is),
    vcastuzxlo := ~.vcastizxlo,
    vcastuzxhi := ~.vcastizxhi,

    average := (self, o, i, is) >> CondPat(o,
        [average, @TVect, @TVect], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
            Cond( sfx in ["epu8", "epu16"],
                self.printf("_mm_avg_$1($2, $3)",[sfx, o.args[1],o.args[2]]),
                Error("finish SSE unparser"))),
        Inherited(o, i, is)),
));


text:
CopyFile := function(src, dst)
  local f, g, line;
  f := InputTextFile(src);
  g := OutputTextFile(dst, false);
  while true do
    line := ReadLine(f);
    if line = fail then
      break
    else
      WriteLine(g, Chomp(line));
    fi;
  od;
  CloseStream(f);
  CloseStream(g);
end;

ext: gap

max_issues_repo_name: seanwallawalla-forks/RosettaCodeData

max_forks_repo_name: seanwallawalla-forks/RosettaCodeData


text:
# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

vtransposelo := (l1,l2,n) -> Flat(List([1..n/2], i-> [l1[2*i-1],l2[2*i-1]]
));

vtransposehi := (l1,l2,n) -> Flat(List([1..n/2], i-> [l1[2*i],l2[2*i]]
));

vrev64 := (l1,n) -> Flat(List([1..n/2], i-> [l1[2*i],l1[2*i-1]]
));

ext: gi


text:
# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

# Below we define includes mandated by SSE Intel C+ compiler
# Due to known incompatibilities between gcc and icc, you can turn
# off hasXXX if compiler doesnt support it
# No crosscompilation support.

_hasSSE4_2 := arg -> LocalConfig.cpuinfo.SIMD().hasSSE4_2() and  LocalConfig.compilerinfo.SIMD().hasSSE4_2();
_hasSSE4_1 := arg -> LocalConfig.cpuinfo.SIMD().hasSSE4_1() and  LocalConfig.compilerinfo.SIMD().hasSSE4_1();
_hasSSSE3  := arg -> LocalConfig.cpuinfo.SIMD().hasSSSE3()  and  LocalConfig.compilerinfo.SIMD().hasSSSE3();
_hasSSE3   := arg -> LocalConfig.cpuinfo.SIMD().hasSSE3()   and  LocalConfig.compilerinfo.SIMD().hasSSE3();
_hasSSE2   := arg -> LocalConfig.cpuinfo.SIMD().hasSSE2()   and  LocalConfig.compilerinfo.SIMD().hasSSE2();
_hasSSE    := arg -> LocalConfig.cpuinfo.SIMD().hasSSE()    and  LocalConfig.compilerinfo.SIMD().hasSSE();
_hasMMX    := arg -> LocalConfig.cpuinfo.SIMD().hasMMX()    and  LocalConfig.compilerinfo.SIMD().hasMMX();

_MM_MALLOC := () -> When(not LocalConfig.osinfo.isDarwin(), ["<include/mm_malloc.h>"], []);
_MMINTRIN  := () -> When(_hasMMX(),    ["<mmintrin.h>"], []);
_XMMINTRIN := () -> When(_hasSSE(),    ["<xmmintrin.h>"], []);
_EMMINTRIN := () -> When(_hasSSE2(),   ["<emmintrin.h>"], []);
_PMMINTRIN := () -> When(_hasSSE3(),   ["<pmmintrin.h>"], []);
_TMMINTRIN := () -> When(_hasSSSE3(),  ["<tmmintrin.h>"], []);
_SMMINTRIN := () -> When(_hasSSE4_1(), ["<smmintrin.h>"], []);
_NMMINTRIN := () -> When(_hasSSE4_2(), ["<nmmintrin.h>"], []);

#F ==============================================================================================
#F SIMD_Intel  --  Base class for Intel ISAs
#F
Class(SIMD_Intel, SIMD_ISA, rec(
    info := "Intel SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, MMX architecture definition base",
    arch := "Intel_SSE",
    file := "sse",
    unparser := SSEUnparser,

    # Should vector values be inlined or declared as constants
    declareConstants := false,

    # This is applied as the final pass of compileStrategy. Below we apply ISA specific strength
    # reduction rules, in particular some unaligned stores can only be eliminated in the final pass
    fixProblems := (self, c, opts) >> BUA(c, MergedRuleSet(RulesStrengthReduce, RulesSSEPostProcess), opts),

    autolib := rec(
	includes := () -> _NMMINTRIN() :: _SMMINTRIN() :: _TMMINTRIN() :: _PMMINTRIN() ::
	                  _EMMINTRIN() :: _XMMINTRIN(),
        timerIncludes := () -> ["<include/sp_rdtsc.h>"]),

    unsigned := meth(self) self.isSigned:=false; return self; end,

    vzero := self >> self.t.zero(),

    intelCommonIncludes := self >> _NMMINTRIN() :: _SMMINTRIN() :: _TMMINTRIN() ::
                                   _PMMINTRIN() :: _EMMINTRIN() :: _XMMINTRIN() :: _MMINTRIN(),

    simpIndicesInside := SSE_LDST.list
));

# ==============================================================================================
# 2-way double precision real
#
Class(SSE_2x64f, SIMD_Intel, rec(
    info := "SSE2 2 x 64-bit double",

    countrec := rec(
        ops := [
            [ add, sub, chslo_2x64f, chshi_2x64f, addsub_2x64f, hadd_2x64f],
	    [ mul ],
            [ vunpacklo_2x64f, vunpackhi_2x64f, vshuffle_2x64f, vushuffle_2x64f ],
            [ vload1sd_2x64f,  vload_1l_2x64f,  vload_1h_2x64f, vloadu_2x64f,
	      vstore_1l_2x64f, vstore_1h_2x64f, vstoreu_2x64f ],
            [ deref ],
            Value      # Value without [] is a keyword in countOps !!
        ],
        printstrings := ["[adds]", "[mults]", "[vperms]", "[svldst]", "[vldst]", "[vval]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]
    ),

    includes     := self >> ["<include/omega64.h>", "<include/mm_malloc.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := false,
    isFloat      := true,

    v     := 2,
    t     := TVect(TReal, 2),
    ctype := "double",
    instr := [vunpacklo_2x64f, vunpackhi_2x64f, vshuffle_2x64f],
    bits  := 64,

    splopts  := rec(precision := "double"),
    dupload  := (y, x) -> assign(y, vdup(x, 2)),
    duploadn := (y, x, n) -> assign(y, vushuffle_2x64f(x, [n,n])),
    hadd     := (x1,x2) -> hadd_2x64f(x1,x2),

    # load full vectors using subvectors of length 1 or 2
    svload:= [ [ (y,x,opts) -> assign(y, vload1sd_2x64f(x[1].toPtr(TReal))),
                 (y,x,opts) -> let(u := var.fresh_t("U", TVectDouble(2)),
                      decl([u], chain(assign(u, vload1sd_2x64f(x[1].toPtr(TReal))),
                                      assign(y, vload_1h_2x64f(u, x[2].toPtr(TReal)))))) ],
               [ (y,x,opts) -> assign(y, nth(x[1].toPtr(TVect(TReal, 2)), 0)) ]
	     ],

    # store full vector using subvectors of length 1 or 2
    svstore := [ [ (y,x,opts) -> vstore_1l_2x64f(y[1].toPtr(TReal), x),
                   (y,x,opts) -> chain(vstore_1l_2x64f(y[1].toPtr(TReal), x),
                                       vstore_1h_2x64f(y[2].toPtr(TReal), x)) ],
                 [ (y,x,opts) -> assign(nth(y[1].toPtr(TVect(TReal, 2)), 0), x) ]
	       ],

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (c, n, opts) -> When(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and not(n=2),
        let(f:="0xFFFFFFFF", z:="0x0", bin_and(c,tcast(TVect(TReal, 2), vhex(List([1..4],x->When(x/2<=n,f,z)))))),
        c),

    # load contiguous with unaligned loads
    loadc := (self, sv, opts) >> (
	(y,x) -> assign(y, self.optional_mask(vloadu_2x64f(x.toPtr(TReal)), sv, opts))),

    # load contiguous + known alignment -> using 2 aligned load to be smarter
    loadc_align := (self, sv, align, opts) >> ((y,x,addr) -> let(
        v1 := nth(nth(x,add(addr,-align)).toPtr(TVect(TReal, 2)),0),
        v2 := nth(nth(x,add(addr,2-align)).toPtr(TVect(TReal, 2)), 0),
        m := x -> self.optional_mask(x, sv, opts),

        Cond(align=0,  assign(y, m(v1)),
             sv=1,     assign(y, m(vushuffle_2x64f(v1, [2, 2]))),
             sv=2,     assign(y, m(vshuffle_2x64f(v1, v2, [2, 3]))),
             Error("bad parameters")))),

    # store contiguous unaligned
    storec := [ (y,x) -> vstore_1l_2x64f(y.toPtr(TReal), x),
                (y,x) -> vstoreu_2x64f  (y.toPtr(TReal), x) ],

    reverse := (y,x) -> assign(vref(y,0,2), vushuffle_2x64f(vref(x,0,2), [2,1])),

    mul_cx := (self, opts) >> Cond(
                    # SSE and MMX can't do double precision
                    opts.vector.SIMD in ["MMX", "SSE"],
                    Error("SSE2 required for double precision"),
                    # SSE2 only
                    opts.vector.SIMD = "SSE2",
                    (y,x,c) -> let(u1 := var.fresh_t("U", TVectDouble(2)), u2 := var.fresh_t("U", TVectDouble(2)),
                        u3 := var.fresh_t("U", TVectDouble(2)), u4 := var.fresh_t("U", TVectDouble(2)),
                        decl([u1, u2, u3, u4], chain(
                            assign(u1, mul(x, vushuffle_2x64f(c, [1,1]))),
                            assign(u2, chshi_2x64f(x)),
                            assign(u3, mul(u2, vushuffle_2x64f(c, [2,2]))),
                            assign(u4, vushuffle_2x64f(u3, [2,1])),
                            assign(y, add(u1, u4))))),
                    # SSE3 or higher
                    (y, x, c) -> let(u1 := var.fresh_t("U", TVectDouble(2)),
                                 u2 := var.fresh_t("U", TVectDouble(2)),
                                 u3 := var.fresh_t("U", TVectDouble(2)),
                        decl([u1, u2, u3], chain(
                            assign(u1, mul(x, vushuffle_2x64f(c, [1,1]))),
                            assign(u2, vushuffle_2x64f(x, [2,1])),
                            assign(u3, mul(u2, vushuffle_2x64f(c, [2,2]))),
                            assign(y, addsub_2x64f(u1, u3)))))
            ),

   mul_cx_conj := (self, opts) >> Cond(
                    # SSE and MMX can't do double precision
                    opts.vector.SIMD in ["MMX", "SSE"],
                    Error("SSE2 required for double precision"),
                    # SSE2 only
                    (y,x,c) -> let(u1 := var.fresh_t("U", TVectDouble(2)), u2 := var.fresh_t("U", TVectDouble(2)),
                        u3 := var.fresh_t("U", TVectDouble(2)), u4 := var.fresh_t("U", TVectDouble(2)),
                        decl([u1, u2, u3, u4], chain(
                            assign(u1, mul(x, vushuffle_2x64f(c, [1,1]))),
                            assign(u2, chslo_2x64f(x)),
                            assign(u3, mul(u2, vushuffle_2x64f(c, [2,2]))),
                            assign(u4, vushuffle_2x64f(u3, [2,1])),
                            assign(y, add(u1, u4)))))
		    ),

    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 1)),
    bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),
    bin_shr2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
    # support for VO1dsJ(n, v)
    bin_shrev := (y,x,opts) -> assign(y, vshuffle_2x64f(x[1], x[2], [1,2])),
    swap_cx := (y, x, opts) -> assign(y, vushuffle_2x64f(x, [2,1])),
    RCVIxJ2 := (y, x, opts) -> assign(y, vushuffle_2x64f(x, [2,1])) # ???? maybe shuf3 rule is invalid

));

#==============================================================================================
#
Class(SSE_2x64i, SIMD_Intel, rec(
    info := "SSE2 2 x 64-bit integer",

    includes     := self >> ["<include/omega64i.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := true,
    isFloat      := false,
    saturatedArithmetic := false,

    v     := 2,
    t     := TVect(TReal, 2),
    ctype := "__int64",
    instr := [vunpacklo_2x64i, vunpackhi_2x64i, vshuffle_2x64i],
    bits  := 64,
    fracbits := 62,

    splopts := rec(precision := "double"),
#    dupload := (y, x) -> assign(y, vushuffle_2x64i(vload1sd_2x64i(x.toPtr(TReal)), [1,1])),
    reverse := (y,x) -> assign(vref(y,0,2), vushuffle_2x64f(vref(x,0,2), [2,1])),
    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 1)),
    bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),
    bin_shr2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
));

#==============================================================================================
Class(SSE_2x32f, SIMD_Intel, rec(
    info := "SSE 2 x 32-bit float",
    countrec := rec(
        ops := [
            [add, sub, addsub_4x32f, hadd_4x32f], [mul],
            [vunpacklo_4x32f, vunpackhi_4x32f, vshuffle_4x32f, vushuffle_4x32f],
            [vload1_4x32f,  vloadu_4x32f,  vloadu2_4x32f,
             vstore1_4x32f, vstoreu_4x32f, vstoreu2_4x32f, vstoremsk_4x32f],
            [deref, vload_2l_4x32f, vload_2h_4x32f, vstore_2l_4x32f, vstore_2h_4x32f],
            Value      # Value without [] is a keyword in countOps !!
        ],
        printstrings := ["[adds]", "[mults]", "[vperms]", "[svldst]", "[vldst]", "[vval]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]
    ),

    includes     := self >> ["<include/omega32.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := false,
    isFloat      := true,

    v      := 2,
    t      := TVect(TReal, 2),
    ctype  := "float",
    instr  := [vunpacklo_2x32f, vunpackhi_2x32f, vshuffle_2x32f],
    bits   := 32,

    splopts  := rec(precision := "single"),
    dupload  := (y, x) -> assign(y, vdup(x, 2)),
    duploadn := (y, x, n) -> assign(y, vushuffle_2x32f(x, [n,n])),
    hadd     := (x1,x2) -> hadd_2x32f(x1,x2),

    # NOTE: Hacks. YSV: these are still alive as of svn 9206.
    #
    requireLoad := true,
    loadop := l -> vload_2x32f(vzero_2x32f(), l),
    requireStore := true,
    storeop := (l, v) -> vstore_2x32f(l.loc, v),
    needScalarVarFix := true,
    scalarVar := () -> var.fresh_t("P", TVectDouble(4)),
    ##

    svload:= [ [     # load full vectors using subvectors of length 1 or 2
                (y,x,opts) -> assign(y, vload1sd_2x32f(x[1].toPtr(TReal))),
                (y,x,opts) -> let(u := var.fresh_t("U", TVectDouble(2)),
                    decl([u], chain(assign(u, vload1sd_2x32f(x[1].toPtr(TReal))),
                                    assign(y, vload_1h_2x32f(u, x[2].toPtr(TReal))))))
                ],
                [(y,x,opts) -> assign(y, nth(x[1].toPtr(TVect(TReal, 2)), 0)) ]
        ],
    svstore := [[    # store full vector using subvectors of length 1 or 2
                (y,x,opts) -> vstore_1l_2x32f(y[1].toPtr(TReal), x),
                (y,x,opts) -> chain(vstore_1l_2x32f(y[1].toPtr(TReal), x),
                               vstore_1h_2x32f(y[2].toPtr(TReal), x))
                ],
                [
                (y,x,opts) -> assign(nth(y[1].toPtr(TVect(TReal, 2)), 0), x)
                ]],

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (self, c, n, opts) >> Cond(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and n<>2,
	bin_and(c, tcast(self.t, vhex(List([1..4], x -> When(x/2<=n, "0xFFFFFFFF", "0x0"))))),
        c),

    # load contiguous with unaligned loads
    loadc := (self, sv, opts) >> (
	(y,x) -> assign(y, self.optional_mask(vloadu_2x32f(x.toPtr(TReal)), sv, opts))),

    # load contiguous + known alignment -> using 2 aligned load to be smarter
    loadc_align := (self, sv, align, opts)>>
    ((y,x,addr) -> let(
        v1 := nth(nth(x,add(addr,-align)).toPtr(TVect(TReal, 2)),0),
        v2 := nth(nth(x,add(addr,2-align)).toPtr(TVect(TReal, 2)), 0),
        m := x-> self.optional_mask(x, sv, opts),
        Cond(align=0,
                 assign(y, m(v1)),
	     sv=1,
                 assign(y, m(vushuffle_2x32f(v1, [2, 2]))),
             sv=2,
                 assign(y, m(vshuffle_2x32f(v1, v2, [2, 3]))),
	     # else
		 Error("bad parameters")))),

    # store contiguous unaligned
    storec := [
        (y,x) -> vstore_1l_2x32f(y.toPtr(TReal), x),
        (y,x) -> vstoreu_2x32f(y.toPtr(TReal), x)
    ],

    reverse := (y,x) -> assign(vref(y,0,2), vushuffle_2x32f(vref(x,0,2), [2,1])),

    mul_cx := (self, opts) >> Cond(
        # SSE and MMX can't do double precision
        opts.vector.SIMD in ["MMX", "SSE"],
            Error("SSE2 required for double precision"),

        # SSE2 only
        opts.vector.SIMD = "SSE2",
            (y,x,c) -> let(
		u := var.fresh_t("U", self.t), w := var.fresh_t("U", self.t),
                decl([u, w], chain(
                     assign(u,             x  * vushuffle_2x32f(c, [1,1])),
                     assign(w, chshi_2x32f(x) * vushuffle_2x32f(c, [2,2])),
                     assign(y, u + vushuffle_2x32f(w, [2,1]))))),
        # SSE3 or higher
            (y, x, c) -> let(
		u := var.fresh_t("U", self.t), w := var.fresh_t("U", self.t),
                decl([u, w], chain(
                     assign(u,                       x   * vushuffle_2x32f(c, [1,1])),
                     assign(w, vushuffle_2x32f(x, [2,1]) * vushuffle_2x32f(c, [2,2])),
                     assign(y, addsub_2x32f(u, w)))))
    ),

    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 8)),
    bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),
    bin_shr2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],1))),
    # support for VO1dsJ(n, v)
    bin_shrev := (y,x,opts) -> assign(y, vshuffle_2x32f(x[1], x[2], [1,2]))
));

#==============================================================================================

Class(SSE_4x32f, SIMD_Intel, rec(
    info := "SSE 4 x 32-bit float",

    # experimental -- parametized ISA, <el_t> is the type of each slot in the vector
    __call__ := (self, el_t) >> WithBases(self, rec(
        t            := TVect(el_t, 4),
	isSigned     := el_t.isSigned(),
	isFloat      := IsRealT(el_t),
	isFixedPoint := IsFixedPtT(el_t),
	splopts      := CopyFields(self.splopts, rec(XType := TPtr(el_t), YType := TPtr(el_t))),
	svload       := self.svload_init(TVect(el_t, 4)),
	svstore      := self.svstore_init(TVect(el_t, 4)),
	storec       := self.storec_init(TVect(el_t, 4)),
	operations   := ISAOps,
	print        := self >> Print(self.__name__, "(", self.t.t, ")"),
	id           := self >> self.__name__ :: "_" :: el_t.strId(),
    )),

    countrec := rec(
        ops := [
            [ add, sub, addsub_4x32f, hadd_4x32f, chshi_4x32f, chslo_4x32f ],
	    [ mul ],
            [ vunpacklo_4x32f, vunpackhi_4x32f, vshuffle_4x32f, vushuffle_4x32f ],
            [ vload1_4x32f,    vload_2l_4x32f,  vload_2h_4x32f,  vloadu_4x32f,  vloadu2_4x32f,
              vstore1_4x32f,   vstore_2l_4x32f, vstore_2h_4x32f, vstoreu_4x32f, vstoreu2_4x32f,
	      vstoremsk_4x32f, vinsert_4x32f,   vextract_4x32f ],
            [ deref ],
            Value,      # Value without [] is a keyword in countOps !!
            [vcvt_4x32_i2f]
        ],
        printstrings := ["[adds]", "[mults]", "[vperms]", "[svldst]", "[vldst]", "[vval]", "[vcvt]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]
    ),

    includes     := self >> ["<include/omega32.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := false,
    isFloat      := true,

    v      := 4,
    t      := TVect(TReal, 4),
    ctype  := "float",
    instr  := [vunpacklo_4x32f, vunpackhi_4x32f, vshuffle_4x32f, vushuffle_4x32f],
    bits   := 32,

    splopts  := rec(precision := "single"),

    #dupload := (self, y, x) >> self.duploadn(y, vload1_4x32f(x.toPtr(self.t.t)), 1),
    dupload := (self, y, x) >> assign(y, vdup(x, self.v)),
    duploadn := (y, x, n) -> assign(y, vushuffle_4x32f(x, [n,n,n,n])),

    hadd     := (x1,x2) -> hadd_4x32f(x1,x2),

    svload_init := (vt) -> [
        # load using subvectors of length 1
	[
            (y,x,opts) -> assign(y, vload1_4x32f(x[1].toPtr(vt.t))),

            (y,x,opts)->When(_hasSSE4_1(opts),
                let(u := var.fresh_t("U", vt),
                    decl([u], chain(
                            assign(u, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(y, vinsert_4x32f(u, deref(x[2].toPtr(T_Int(32))), 2))
                            ))),
                let(u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
                    decl([u1, u2], chain(
                            assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(u2, vload1_4x32f(x[2].toPtr(vt.t))),
                            assign(y, vunpacklo_4x32f(u1, u2))
                            )))
                ),

            (y,x,opts)->When(_hasSSE4_1(opts),
                let(u := var.fresh_t("U", vt),
                    decl([u], chain(
                            assign(u, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(u, vinsert_4x32f(u, deref(x[2].toPtr(T_Int(32))), 2)),
                            assign(y, vinsert_4x32f(u, deref(x[3].toPtr(T_Int(32))), 3))
                            ))),
                let(u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
                    u3 := var.fresh_t("U", vt), u4 := var.fresh_t("U", vt),
                    decl([u1, u2, u3, u4], chain(
                            assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(u2, vload1_4x32f(x[2].toPtr(vt.t))),
                            assign(u3, vshuffle_4x32f(u1, u2, [1,2,1,2])),
                            assign(u4, vload1_4x32f(x[3].toPtr(vt.t))),
                            assign(y, vshuffle_4x32f(u3, u4, [1,3,1,3]))
                            )))
                ),

            (y,x,opts)->When(_hasSSE4_1(opts),
                let(u := var.fresh_t("U", vt),
                    decl([u], chain(
                            assign(u, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(u, vinsert_4x32f(u, deref(x[2].toPtr(T_Int(32))), 2)),
                            assign(u, vinsert_4x32f(u, deref(x[3].toPtr(T_Int(32))), 3)),
                            assign(y, vinsert_4x32f(u, deref(x[4].toPtr(T_Int(32))), 4))
                            ))),
                let(u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
                    u3 := var.fresh_t("U", vt), u4 := var.fresh_t("U", vt),
                    u5 := var.fresh_t("U", vt), u6 := var.fresh_t("U", vt),
                    decl([u1, u2, u3, u4, u5, u6], chain(
                            assign(u1, vload1_4x32f(x[1].toPtr(vt.t))),
                            assign(u2, vload1_4x32f(x[2].toPtr(vt.t))),
                            assign(u3, vshuffle_4x32f(u1, u2, [1,2,1,2])),
                            assign(u4, vload1_4x32f(x[3].toPtr(vt.t))),
                            assign(u5, vload1_4x32f(x[4].toPtr(vt.t))),
                            assign(u6, vshuffle_4x32f(u4, u5, [1,2,1,2])),
                            assign(y,  vshuffle_4x32f(u3, u6, [1,3,1,3]))
                            )))
                ),
	],
        # load using subvectors of length 2
        [
            (y,x,opts) -> assign(y, vload_2l_4x32f(vzero_4x32f(), x[1].toPtr(TVect(vt.t,2)))),
            (y,x,opts) -> let(u := var.fresh_t("U", vt),
                decl(u, chain(
                        assign(u, vload_2l_4x32f(vzero_4x32f(), x[1].toPtr(TVect(vt.t,2)))),
                        assign(y, vload_2h_4x32f(u, x[2].toPtr(TVect(vt.t,2)))))
                    ))
        ]],

    svload := ~.svload_init(~.t),

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (c, n, opts) -> Cond(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and n <> 4,
	bin_and(c, vhex(List([1..4], x -> When(x<=n, "0xFFFFFFFF", "0x0")))),
        c),

    # load contiguous with unaligned loads
    loadc := (self, sv, opts) >> ((y,x) -> assign(y, self.optional_mask(vloadu_4x32f(x.toPtr(self.t.t)), sv, opts))),

    #load contiguous + known alignment -> using 2 aligned load to be smarter
    loadc_align := (self, sv, align, opts)>>
    ((y,x,addr) -> let(
        v1 := nth(nth(x, addr-align)  .toPtr(self.t), 0),
        v2 := nth(nth(x, addr-align+4).toPtr(self.t), 0),
        m := x -> self.optional_mask(x, sv, opts),
        Cond(align=0,
            assign(y, m(v1)),

            _hasSSSE3(opts),
            assign(y, m(alignr_4x32f(v2, v1, align*4))),

            assign(y, m(bin_or(
                        vec_shr(v1, align),
                        vec_shl(v2, (4-align))
                        )))))),

    svstore_init := (vt) -> [
        [
            (y,x,opts) -> vstore1_4x32f(y[1].toPtr(vt.t), x),
            (y,x,opts)->When(_hasSSE4_1(opts),
                chain(
                    vstore1_4x32f(y[1].toPtr(vt.t), x),
                    vextract_4x32f(y[2].toPtr(T_Int(32)), x, 2)
                    ),
                let(u1 := var.fresh_t("U", vt),
                    decl([u1], chain(
                            vstore1_4x32f(y[1].toPtr(vt.t), x),
                            assign(u1, vushuffle_4x32f(x, [2,3,4,1])),
                            vstore1_4x32f(y[2].toPtr(vt.t), u1)
                            )))
                ),
            (y,x,opts)->When(_hasSSE4_1(opts),
                chain(
                    vstore1_4x32f(y[1].toPtr(vt.t), x),
                    vextract_4x32f(y[2].toPtr(T_Int(32)), x, 2),
                    vextract_4x32f(y[3].toPtr(T_Int(32)), x, 3)
                    ),
                let(u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
                    decl([u1, u2], chain(
                            vstore1_4x32f(y[1].toPtr(vt.t), x),
                            assign(u1, vushuffle_4x32f(x, [2,3,4,1])),
                            vstore1_4x32f(y[2].toPtr(vt.t), u1),
                            assign(u2, vushuffle_4x32f(u1, [2,3,4,1])),
                            vstore1_4x32f(y[3].toPtr(vt.t), u2)
                            )))
                ),
            (y,x,opts)->When(_hasSSE4_1(opts),
                chain(
                    vstore1_4x32f(y[1].toPtr(vt.t), x),
                    vextract_4x32f(y[2].toPtr(T_Int(32)), x, 2),
                    vextract_4x32f(y[3].toPtr(T_Int(32)), x, 3),
                    vextract_4x32f(y[4].toPtr(T_Int(32)), x, 4)
                    ),
                let(u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
                    u3 := var.fresh_t("U", vt),
                    decl([u1, u2, u3], chain(
                            vstore1_4x32f(y[1].toPtr(vt.t), x),
                            assign(u1, vushuffle_4x32f(x, [2,3,4,1])),
                            vstore1_4x32f(y[2].toPtr(vt.t), u1),
                            assign(u2, vushuffle_4x32f(u1, [2,3,4,1])),
                            vstore1_4x32f(y[3].toPtr(vt.t), u2),
                            assign(u3, vushuffle_4x32f(u2, [2,3,4,1])),
                            vstore1_4x32f(y[4].toPtr(vt.t), u3)
                            )))
                )
        ],
        [
            (y,x,opts) -> vstore_2l_4x32f(y[1].toPtr(TVect(vt.t,2)), x),
            (y,x,opts) -> chain(vstore_2l_4x32f(y[1].toPtr(TVect(vt.t,2)), x),
                                vstore_2h_4x32f(y[2].toPtr(TVect(vt.t,2)), x))
        ]
    ],

    svstore := ~.svstore_init(~.t),

    # Store contiguous unaligned
    storec_init := (vt) -> [
        (y,x) -> vstore1_4x32f(y.toPtr(vt.t), x),
        (y,x) -> vstoreu2_4x32f(y.toPtr(TVect(T_Int(64),2)), x),
        (y,x) -> vstoremsk_4x32f(y.toPtr(vt.t), x, ["0xFFFFFFFF", "0xFFFFFFFF", "0xFFFFFFFF", "0x0"]),
        (y,x) -> vstoreu_4x32f(y.toPtr(vt.t), x)
    ],

    storec := ~.storec_init(~.t),

    reverse := (y,x) -> assign(vref(y,0,4), vushuffle_4x32f(vref(x,0,4), [4,3,2,1])),

    # support for VS and VS.transpose()
    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 1)),
#   bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],3), vec_shl(x[2],1))),
    bin_shl2 := (self,y,x,opts) >> When(
	_hasSSSE3(opts),
            assign(y, alignr_4x32f(x[2], x[1], 12)),
	# else
        let(u := self.freshU(),
            chain(
                assign(u, vshuffle_4x32f(x[1], x[2], [3,4,1,2])),
                assign(y, vshuffle_4x32f(u,    x[2], [2,3,2,3]))))
    ),

    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),

    bin_shr2 := (self,y,x,opts) >> Cond(
	_hasSSSE3(opts),
            assign(y, alignr_4x32f(x[2], x[1], 4)),
	# else, no SSSE3
            let(u := self.freshU(), chain(                           # x = [a,b,c,d] [e,f,g,h]
                assign(u, vshuffle_4x32f(x[1], x[2], [3,4,1,2])),    # u = [c,d,e,f]
                assign(y, vshuffle_4x32f(x[1], u,    [2,3,2,3]))))), # y = [b,c,d,e]
	# another way
	#   assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],3)))

    # support for VO1dsJ(n, v)
    bin_shrev := (self, y, x, opts) >> let(
	u := self.freshU(),                                     # data = [a b c d] [e f g h]
        chain(                                                  # x = [e,f,g,h] [a,b,c,d]
            assign(u, vshuffle_4x32f(x[1], x[2], [1,2,3,4])),   # u = [e f c d]
            assign(y, vshuffle_4x32f(u,    x[2], [1,4,3,2])))), # y = [e d c b]

    # returns function (y,x,c) -> <code for y=x*c>
    mul_cx := (self, opts) >> Cond(
        # MMX can't do single precision
        opts.vector.SIMD in ["MMX"], Error("SSE required for single precision"),
        # SSE only
        opts.vector.SIMD in ["SSE", "SSE2"],
            (y,x,c) -> let(
		u := self.freshU(),  v := self.freshU(),
                decl([u, v], chain(
                    assign(u,                             x * vushuffle_4x32f(c, [1,1,3,3])),
                    assign(v, vushuffle_4x32f(x, [2,1,4,3]) * vushuffle_4x32f(c, [2,2,4,4])),
                    assign(y, u + chslo_4x32f(v))))),
        # SSE3 or higher
            (y,x,c) -> let(
		u := self.freshU(),  v := self.freshU(),
                decl([u, v], chain(
                    assign(u,                             x * vushuffle_4x32f(c, [1,1,3,3])),
                    assign(v, vushuffle_4x32f(x, [2,1,4,3]) * vushuffle_4x32f(c, [2,2,4,4])),
                    assign(y, addsub_4x32f(u, v)))))
    ),

    # returns function (y,x,c) -> <code for y=x*conj(c)>
    mul_cx_conj := (self, opts) >> Cond(
        # MMX can't do single precision
        opts.vector.SIMD in ["MMX"], Error("SSE required for single precision"),
        # SSE
            (y,x,c) -> let(
		u := self.freshU(),  v := self.freshU(),
                decl([u, v], chain(
                    assign(u,             x  * vushuffle_4x32f(c, [1,1,3,3])),
                    assign(v, chslo_4x32f(x) * vushuffle_4x32f(c, [2,2,4,4])),
                    assign(v, vushuffle_4x32f(v, [2,1,4,3])),
                    assign(y, u + v))))
    ),

    swap_cx := (y, x, opts) -> assign(y, vushuffle_4x32f(x, [2,1,4,3])),
    RCVIxJ2 := (y, x, opts) -> assign(y, vushuffle_4x32f(x, [3,4,1,2])),

    freshU  := self >> var.fresh_t("U", self.t),

    hmin := (self, y, x, opts) >> let( u := self.freshU(),
       decl( [u], chain(
           assign(u, min(x, vushuffle_4x32f(x, [2,1,4,3]))),
           assign(y, min(u, vushuffle_4x32f(u, [3,4,1,2])))
       ))),

    rotate_left := (self, shift) >> Cond(
            shift mod 4=0,
                ((y, x) -> assign(vtref(self.t, y, 0), vtref(self.t, x, 0))),
            shift mod 4=1,
                ((y, x) -> assign(vtref(self.t, y, 0), vushuffle_4x32f(vtref(self.t, x, 0), [4,1,2,3]))),
            shift mod 4=2,
                ((y, x) -> assign(vtref(self.t, y, 0), vushuffle_4x32f(vtref(self.t, x, 0), [3,4,1,2]))),
            shift mod 4=3,
                ((y, x) -> assign(vtref(self.t, y, 0), vushuffle_4x32f(vtref(self.t, x, 0), [2,3,4,1]))),
            #else
                ((y, x) -> assign(vtref(self.t, y, 0),
                    bin_or( vec_shl(vtref(self.t, x, 0), imod(shift,4)),
                            vec_shr(vtref(self.t, x, 0), 4-imod(shift,4)))))),
));

#==============================================================================================

Class(SSE_4x32i, SIMD_Intel, rec(
    active := true,
    isFixedPoint := true,
    isFloat := false,
    saturatedArithmetic := false,
    info := "SSE 4 x 32-bit integer",
    v := 4,
    t := TVect(TReal, 4),
    instr := [vunpacklo_4x32i, vunpackhi_4x32i, vshuffle_4x32i, vushuffle_4x32i],
    bits := 32,
    fracbits := 30,
    includes := self >> ["<include/omega32i.h>"] :: self.intelCommonIncludes(),
    splopts := DataTypes.i32re,

#    dupload := (y, x) -> assign(y, vushuffle_4x32i(vload1_4x32i(x.toPtr(TReal)), [1,1,1,1])),
    reverse := (y,x) -> assign(vref(y,0,4), vushuffle_4x32i(vref(x,0,4), [4,3,2,1])),
    # support for VS and VS.transpose()
    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 1)),
    bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],3), vec_shl(x[2],1))),
    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),
    bin_shr2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],3))),
    interleavedmask := (b,idx,x1,x2) -> assign(nth(tcast(TPtr(TUChar), b),idx),interleavedmask_4x32i(x1,x2)),
    vrshift := "_mm_srli_epi32",
    vlshift := "_mm_slli_epi32",
    vmul := "_mm_mulhi_epi32"
));

#==============================================================================================

Class(SSE_8x16i, SIMD_Intel, rec(
    info := "SSE2 8 x 16-bit int",

    # experimental -- parametized ISA, <el_t> is the type of each 8-bit slot in the vector
    __call__ := (self, el_t) >> WithBases(self, rec(
        t            := TVect(el_t, 8),
	isSigned     := el_t.isSigned(),
	isFloat      := IsRealT(el_t),
	isFixedPoint := IsFixedPtT(el_t),
	splopts      := CopyFields(self.splopts, rec(XType := TPtr(el_t), YType := TPtr(el_t))),
	svload       := self.svload_init(TVect(el_t, 8)),
	svstore      := self.svstore_init(TVect(el_t, 8)),
	storec       := self.storec_init(TVect(el_t, 8)),
        operations   := ISAOps,
	print        := self >> Print(self.__name__, "(", self.t.t, ")"),
	id           := self >> self.__name__ :: "_" :: el_t.strId(),
    )),

    countrec := rec(
        ops := [
            [ add, sub, chs_8x16i ],
	    [ fpmul ],
            [ vunpacklo_8x16i,   vunpackhi_8x16i,  vunpacklo2_8x16i, vunpackhi2_8x16i,
              vunpacklo4_8x16i,  vunpackhi4_8x16i, vushuffle2_8x16i, vushufflelo_8x16i,
	      vushufflehi_8x16i, vushuffle_8x16i,  vshuffle2_8x16i,  vshuffle4_8x16i,
	      alignr_8x16i ],
            [ vload1_8x16i, vload2_8x16i, vload4_8x16i, vloadu_8x16i,
              vextract1_8x16i, vextract2_8x16i, vstoreu_8x16i, vstore4_8x16i, vstoremsk_8x16i ],
            [ deref ],
            Value
        ],
        printstrings := ["[adds]", "[mults]", "[vperms]", "[svldst]", "[vldst]", "[vval]"],
        arithcost := (self, opcount) >> opcount[1]+opcount[2]*2
    ),

    includes     := self >> ["<include/omega16i.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := true,
    isFloat      := false,
    saturatedArithmetic := false,

    v     := 8,
    t     := TVect(TReal, 8),
    ctype := "short int",
    instr := [vunpacklo_8x16i,   vunpackhi_8x16i,   vunpacklo2_8x16i, vunpackhi2_8x16i,
              vunpacklo4_8x16i,  vunpackhi4_8x16i,  vushuffle2_8x16i, vushufflelo_8x16i,
	      vushufflehi_8x16i, vshuffle2_8x16i ], #vshuffle4_8x16i
    bits     := 16,
    fracbits := 14,
    vrshift  := "_mm_srai_epi16", # "_mm_srli_epi16",
    vlshift  := "_mm_slli_epi16",

    splopts := rec(customDataType := "short_fp14"),

    dupload := (self, y, x) >> assign(y, vdup(x, self.v)),

    svload_init := (vt) -> [
        # ------------------------------------
        # Load using subvectors of length 1
	[
	  (y,x,opts) -> assign(y, vload1_8x16i(vt.zero(), x[1], 0)),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..2], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..3], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..4], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..5], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..6], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..7], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
          (y,x,opts) -> let(
	      u  := var.fresh_t("U", vt),
	      cc := List([1..8], i -> assign(u, vload1_8x16i(u, x[i], i-1))),
              decl([u], chain(assign(u, vt.zero()), cc, assign(y, u)))),
        ],

        # ------------------------------------
        # Load using subvectors of length 2
        [ (y,x,opts) -> let(
	      u := var.fresh_t("U", vt),
              decl([u], chain(
                      assign(u, vload2_8x16i(nth(x[1].toPtr(T_Int(32)), 0))),
                      assign(u, vunpacklo2_8x16i(u, vt.zero())),
                      assign(y, vunpacklo4_8x16i(u, vt.zero()))))),
          (y,x,opts) -> let(
	      u := var.fresh_t("U", vt), v := var.fresh_t("U", vt),
              decl([u, v], chain(
                      assign(u, vload2_8x16i(nth(x[1].toPtr(T_Int(32)), 0))),
		      assign(v, vload2_8x16i(nth(x[2].toPtr(T_Int(32)), 0))),
                      assign(u, vunpacklo2_8x16i(u, v)),
                      assign(y, vunpacklo4_8x16i(u, vt.zero()))))),
          (y,x,opts) -> let(
	      u := var.fresh_t("U", vt), v := var.fresh_t("U", vt),
	      w := var.fresh_t("U", vt),
              decl([u, v, w], chain(
                      assign(u, vload2_8x16i(nth(x[1].toPtr(T_Int(32)), 0))),
		      assign(v, vload2_8x16i(nth(x[2].toPtr(T_Int(32)), 0))),
                      assign(u, vunpacklo2_8x16i(u, v)),
                      assign(w, vload2_8x16i(nth(x[3].toPtr(T_Int(32)), 0))),
                      assign(w, vunpacklo2_8x16i(w, vt.zero())),
                      assign(y, vunpacklo4_8x16i(u, w))))),
          (y,x,opts) -> let(
	      u := var.fresh_t("U", vt),  v := var.fresh_t("U", vt),
	      uu := var.fresh_t("U", vt), vv := var.fresh_t("U", vt),
              decl([u, v, uu, vv], chain(
                      assign(u, vload2_8x16i(nth(x[1].toPtr(T_Int(32)), 0))),
		      assign(v, vload2_8x16i(nth(x[2].toPtr(T_Int(32)), 0))),
                      assign(u, vunpacklo2_8x16i(u, v)),
                      assign(uu, vload2_8x16i(nth(x[3].toPtr(T_Int(32)), 0))),
		      assign(vv, vload2_8x16i(nth(x[4].toPtr(T_Int(32)), 0))),
                      assign(uu, vunpacklo2_8x16i(uu, vv)),
                      assign(y,  vunpacklo4_8x16i(u, uu)))))
        ]
    ],

    svload := ~.svload_init(~.t),

    svstore_init := (vt) -> [
        # ------------------------------------
        # Store subvectors of length 1
	#
	[ (y,x,opts) -> chain(List([1..1], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..2], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..3], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..4], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..5], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..6], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..7], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
          (y,x,opts) -> chain(List([1..8], i -> assign(y[i], vextract1_8x16i(x, i-1)))),
        ],
        # ------------------------------------
        # Store subvectors of length 2
	#
        [ (y,x,opts) -> assign(nth(y[1].toPtr(T_Int(32)),0), vextract2_8x16i(x)),

          (y,x,opts) -> let(
	      u := var.fresh_t("U", vt),
              decl([u], chain(
                      assign(nth(y[1].toPtr(T_Int(32)),0), vextract2_8x16i(x)),
                      assign(u, vushuffle2_8x16i(x, [2,3,4,1])),
		      assign(nth(y[2].toPtr(T_Int(32)),0), vextract2_8x16i(u))
                      ))),
          (y,x,opts) -> let(
	      u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt),
              decl([u1, u2], chain(
                      assign(nth(y[1].toPtr(T_Int(32)),0), vextract2_8x16i(x)),
                      assign(u1, vushuffle2_8x16i(x,  [2,3,4,1])), assign(nth(y[2].toPtr(T_Int(32)),0), vextract2_8x16i(u1)),
                      assign(u2, vushuffle2_8x16i(u1, [2,3,4,1])), assign(nth(y[3].toPtr(T_Int(32)),0), vextract2_8x16i(u2))
                      ))),
          (y,x,opts) -> let(
	      u1 := var.fresh_t("U", vt), u2 := var.fresh_t("U", vt), u3 := var.fresh_t("U", vt),
              decl([u1, u2, u3], chain(
                      assign(nth(y[1].toPtr(T_Int(32)),0), vextract2_8x16i(x)),
                      assign(u1, vushuffle2_8x16i(x,  [2,3,4,1])), assign(nth(y[2].toPtr(T_Int(32)),0), vextract2_8x16i(u1)),
                      assign(u2, vushuffle2_8x16i(u1, [2,3,4,1])), assign(nth(y[3].toPtr(T_Int(32)),0), vextract2_8x16i(u2)),
                      assign(u3, vushuffle2_8x16i(u2, [2,3,4,1])), assign(nth(y[4].toPtr(T_Int(32)),0), vextract2_8x16i(u3))
                      )))
        ]
    ],

    svstore := ~.svstore_init(~.t),

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (c, n, opts) -> Cond(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and n<>8,
        bin_and(c, vhex(List([1..8], x -> When(x<=n, "0xFFFF", "0x0")))),
        c),

    # Load contiguous with unaligned loads
    loadc := (self, sv, opts) >> (
	(y,x) -> assign(y, self.optional_mask(vloadu_8x16i(x.toPtr(self.t.t)), sv, opts))),

    # Load contiguous + known alignment -> using 2 aligned load to be smarter
    loadc_align := (self, sv, align, opts) >>
    ((y,x,addr) -> let(
        v1 := nth(nth(x, addr-align  ).toPtr(self.t), 0),
        v2 := nth(nth(x, addr-align+8).toPtr(self.t), 0),
        mask := x -> self.optional_mask(x, sv, opts),
        Cond(align=0,
                 assign(y, mask(v1)),
	     _hasSSSE3(opts),
                 assign(y, mask(alignr_8x16i(v2, v1, align*2))),
	     # else
                 assign(y, mask(bin_or(vec_shr(v1, align), vec_shl(v2, 8-align))))
	))),

    # Store contiguous unaligned
    storec_init := (vt) -> [
        (y,x) -> assign(y, vextract1_8x16i(x, 0)),
        (y,x) -> assign(nth(y.toPtr(T_Int(32)),0), vextract2_8x16i(x)),
        (y,x) -> vstoremsk_8x16i(y.toPtr(vt.t), x, ["0xFFFF", "0xFFFF", "0xFFFF", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstore4_8x16i(y.toPtr(vt), x),
        (y,x) -> vstoremsk_8x16i(y.toPtr(vt.t), x, ["0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_8x16i(y.toPtr(vt.t), x, ["0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0x0", "0x0"]),
        (y,x) -> vstoremsk_8x16i(y.toPtr(vt.t), x, ["0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0xFFFF", "0x0"]),
        (y,x) -> vstoreu_8x16i(y.toPtr(vt), x)
    ],

    storec := ~.storec_init(~.t),

    reverse := (y,x) -> assign(vref(y,0,8),
	vushufflehi_8x16i(vushufflelo_8x16i(vushuffle2_8x16i(vref(x,0,8), [3,4,1,2]), [4,3,2,1]), [4,3,2,1])),

    # support for VS and VS.transpose()
    bin_shl1 := (self,y,x,opts) >> assign(y, vec_shl(x, 1)),
    bin_shl2 := (self,y,x,opts) >> When(_hasSSSE3(opts),
        assign(y, alignr_8x16i(x[2], x[1], 14)),
        assign(y, bin_or(vec_shr(x[1], 7), vec_shl(x[2], 1)))
    ),

    bin_shr1 := (self,y,x,opts) >> assign(y, vec_shr(x, 1)),
    bin_shr2 := (self,y,x,opts) >> When(_hasSSSE3(opts),
        assign(y, alignr_8x16i(x[2], x[1], 2)),
        assign(y, bin_or(vec_shr(x[1], 1), vec_shl(x[2], 7)))
    ),

    interleavedmask := (b,idx,x1,x2) -> assign(nth(tcast(TPtr(TSym("short int")), b),idx),interleavedmask_8x16i(x1,x2)),

    # support for VO1dsJ(n, v)
    bin_shrev := (y,x,opts) -> let(
	u := var.fresh_t("U", TVectDouble(8)),
        chain(
            opts.vector.isa.bin_shr2(u, [x[2], x[1]], opts),
            assign(y, vushufflehi_8x16i(vushufflelo_8x16i(vushuffle2_8x16i(u, [3,4,1,2]), [4,3,2,1]), [4,3,2,1])))
    ),

    mul_cx := (self, opts) >> Cond(
        # MMX and SSE can't do 8x16i
        opts.vector.SIMD in ["MMX", "SSE"],
            Error("SSE2 required for 8-way 16-bit integer"),

        opts.vector.SIMD in ["SSE2", "SSE3"],
            (y,x,c) -> let(
		u1 := var.fresh_t("U", self.t), u2 := var.fresh_t("U", self.t),
                u3 := var.fresh_t("U", self.t), u4 := var.fresh_t("U", self.t),
                decl([u1, u2, u3, u4], chain(
                        assign(u1, x * vushufflehi_8x16i(vushufflelo_8x16i(c, [1,1,3,3]), [1,1,3,3])),
                        assign(u2, x * self.t.value([1,-1,1,-1,1,-1,1,-1])),
                        assign(u3, u2 * vushufflehi_8x16i(vushufflelo_8x16i(c, [2,2,4,4]), [2,2,4,4])),
                        assign(u4, vushufflehi_8x16i(vushufflelo_8x16i(u3, [2,1,4,3]), [2,1,4,3])),
                        assign(y, add(u1, u4))))),
        # SSSE3 and higher
            (y,x,c) -> let(
		u1 := var.fresh_t("U", self.t), u2 := var.fresh_t("U", self.t),
                u3 := var.fresh_t("U", self.t), u4 := var.fresh_t("U", self.t),
                decl([u1, u2, u3, u4], chain(
                        assign(u1, x * vushuffle_8x16i(c, TVect(T_Int(8), 16).value([0,1,0,1, 4,5,4,5, 8,9,8,9, 12,13,12,13]))),
                        assign(u2, chs_8x16i(x, TVect(T_Int(16), 8).value([1,-1,1,-1,1,-1,1,-1]))),
                        assign(u3, u2 * vushuffle_8x16i(c, TVect(T_Int(8), 16).value([2,3,2,3, 6,7,6,7, 10,11,10,11, 14,15,14,15]))),
                        assign(u4, vushuffle_8x16i(u3, TVect(T_Int(8), 16).value([2,3,0,1, 6,7,4,5, 10,11,8,9, 14,15,12,13]))),
                        assign(y,  u1 + u4))))
    ),

    swap_cx := (y, x, opts) -> assign(y, vushuffle_8x16i(x, TVect(T_Int(8), 16).value([2,1,4,3, 6,5,8,7, 10,9,12,11, 14,13,16,15]))),
    RCVIxJ2 := (y, x, opts) -> assign(y, vushuffle_8x16i(x, TVect(T_Int(8), 16).value([4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11]))),

    # computes the horizontal minimum of the vector and splats it
    hmin := (self,y,x,opts) >> Cond(
        # MMX and SSE can't do 8x16i
        opts.vector.SIMD in ["MMX", "SSE"],
            Error("SSE2 required for 8-way 16-bit integer"),

        opts.vector.SIMD in ["SSE2", "SSE3", "SSSE3"], let(
	    z := var.fresh_t("m", self.t),
	    decl([z], chain(
	        assign(z, Cond(self.t.isSigned(), x, add(x, self.t.value(-128)))),
	        assign(z, min(z, vushufflelo_8x16i(vushufflehi_8x16i(z, [2,1,4,3]), [2,1,4,3]))),
	        assign(z, min(z, vushuffle_4x32i(z, [2,1,4,3]))),
	        assign(z, min(z, vushuffle_4x32i(z, [3,4,1,2]))),
	        assign(y, Cond(self.t.isSigned(), z, add(z, self.t.value(-128))))))),
	# else SSE4 - there is hmin instruction, need to put it here
	let( z := var.fresh_t("m", self.t),
	    decl([z], chain(
	        assign(z, min(x, vushufflelo_8x16i(vushufflehi_8x16i(x, [2,1,4,3]), [2,1,4,3]))),
	        assign(z, min(z, vushuffle_4x32i(z, [2,1,4,3]))),
	        assign(y, min(z, vushuffle_4x32i(z, [3,4,1,2]))))))
    ),

));

#==============================================================================================

_svload_16x8i_sv1 := (cnt) -> (
	(y,x,opts) -> let(
	      u := var.fresh_t("U", y.t),
	      i := Ind(idiv(cnt, 2)),
	      c := [assign(u, vzero_8x16i())]
	           :: List( [1..IntDouble(cnt/2.0)], i->assign(u, vload1_8x16i(u, bin_or(x[2*i-1],  vec_shl(x[2*i],  1)), i-1)) )
	           :: When( cnt mod 2 = 0, [], [assign(u, vload1_8x16i(u, x[cnt]), idiv(cnt, 2))] ),
              decl([u], chain(c, assign(y, u)))
        )
);

_svstore_16x8i_sv1 := (cnt) -> (
	(y,x,opts) -> let(
	      u := var.fresh_t("U", T_UInt(16)),
	      c := ConcatList( [1..IntDouble(cnt/2.0)], i ->
	               [ assign(u, vextract1_8x16i(x, i-1)),
	                 assign(y[2*i-1], u),
	                 assign(y[2*i  ], vec_shr(u, 1)) ] )
	           :: When( cnt mod 2 = 0, [],
	               [ assign(u, vextract1_8x16i(x, idiv(cnt,2))),
	                 assign(y[cnt], u) ]),
              decl([u], chain(c))
        )
);

Class(SSE_16x8i, SIMD_Intel, rec(
    info := "SSE2 16 x 8-bit int",

    # experimental -- parametized ISA, <el_t> is the type of each 8-bit slot in the vector
    __call__ := (self, el_t) >> WithBases(self, rec(
        t            := TVect(el_t, 16),
	isSigned     := el_t.isSigned(),
	isFloat      := IsRealT(el_t),
	isFixedPoint := IsFixedPtT(el_t),
	splopts      := CopyFields(self.splopts, rec(XType := TPtr(el_t), YType := TPtr(el_t))),
	svload       := self.svload_init(TVect(el_t, 16)),
	svstore      := self.svstore_init(TVect(el_t, 16)),
	storec       := self.storec_init(TVect(el_t, 16)),
        operations   := ISAOps,
	print        := self >> Print(self.__name__, "(", self.t.t, ")"),
	id           := self >> self.__name__ :: "_" :: el_t.strId(),
    )),

    includes     := self >> ["<include/omega8i.h>"] :: self.intelCommonIncludes(),
    active       := true,
    isFixedPoint := true,
    isFloat      := false,

    isSigned            := true,
    saturatedArithmetic := false,

    v     := 16,
    t     := TVect(TReal, 16),
    ctype := "__int8",
    instr := [vunpacklo_16x8i,  vunpackhi_16x8i,    vunpacklo2_16x8i, vunpackhi2_16x8i,
              vunpacklo4_16x8i, vunpackhi4_16x8i,   vunpacklo8_16x8i, vunpackhi8_16x8i,
              vushuffle4_16x8i, vushufflelo2_16x8i, vushufflehi2_16x8i,
	      vshuffle4_16x8i,  vshuffle8_16x8i],
    bits     := 8,
    fracbits := 6,
    vrshift := "_mm_srli_epi8",
    vlshift := "_mm_slli_epi8",
    splopts := rec(customDataType := "char_fp4"),

    # NOTE: missing .loadCont, .storeCont, .svstore

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (c, n, opts) -> When(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and not(n=16),
        let(f:="0xFF", z:="0x0", bin_and(c,vhex(List([1..16],x->When(x<=n,f,z))))),
        c),

    loadCont := (self, n, y, yofs, x, xofs, xofs_align, opts) >> let(
	a := _unwrap(xofs_align),
	nn := _unwrap(n),
	yy := vtref(self.t, y, yofs),
	assign(yy, self.optional_mask(vloadu_16x8i(x + xofs), n, opts))),

#    ((y,x,addr) -> let(
#        v1 := nth(nth(x, addr-align  ).toPtr(TVectDouble(8)), 0),
#        v2 := nth(nth(x, addr-align+8).toPtr(TVectDouble(8)), 0),
#        mask := x -> self.optional_mask(x, sv, opts),
#        Cond(align=0,
#                 assign(y, mask(v1)),
#	     _hasSSSE3(opts),
#                 assign(y, mask(alignr_8x16i(v2, v1, align*2))),
#	     # else
#                 assign(y, mask(bin_or(vec_shr(v1, align),
#                                       vec_shl(v2, (8-align))))))
#	)),

    # NOTE: rewrite using a method, and add missing.
    svload_init := (vt) -> [
	[ _svload_16x8i_sv1(1),
	  _svload_16x8i_sv1(2),
	  _svload_16x8i_sv1(3),
	  _svload_16x8i_sv1(4),
	  _svload_16x8i_sv1(5),
	  _svload_16x8i_sv1(6),
	  _svload_16x8i_sv1(7),
	  _svload_16x8i_sv1(8),
	  _svload_16x8i_sv1(9),
	  _svload_16x8i_sv1(10),
	  _svload_16x8i_sv1(11),
	  _svload_16x8i_sv1(12),
	  _svload_16x8i_sv1(13),
	  _svload_16x8i_sv1(14),
	  _svload_16x8i_sv1(15),
	  _svload_16x8i_sv1(16)
        ]
    ],

    svload := ~.svload_init(~.t),

    svstore_init := (vt) -> [
	[ _svstore_16x8i_sv1(1),
	  _svstore_16x8i_sv1(2),
	  _svstore_16x8i_sv1(3),
	  _svstore_16x8i_sv1(4),
	  _svstore_16x8i_sv1(5),
	  _svstore_16x8i_sv1(6),
	  _svstore_16x8i_sv1(7),
	  _svstore_16x8i_sv1(8),
	  _svstore_16x8i_sv1(9),
	  _svstore_16x8i_sv1(10),
	  _svstore_16x8i_sv1(11),
	  _svstore_16x8i_sv1(12),
	  _svstore_16x8i_sv1(13),
	  _svstore_16x8i_sv1(14),
	  _svstore_16x8i_sv1(15),
	  _svstore_16x8i_sv1(16)
        ]
    ],

    svstore := ~.svstore_init(~.t),

    bin_shl1 := (y,x,opts) -> assign(y, vec_shl(x, 1)),
    bin_shl2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],15), vec_shl(x[2],1))),
    bin_shr1 := (y,x,opts) -> assign(y, vec_shr(x, 1)),
    bin_shr2 := (y,x,opts) -> assign(y, bin_or(vec_shr(x[1],1), vec_shl(x[2],15))),

    bin_shr  := (self, y, x, shift, opts) >> assign(y, bin_and( tcast(self.t, bin_shr(tcast(TVect(T_Int(16), 8), x), shift)),
	                                                        TVect(T_UInt(8), 16).value(idiv(255, 2^shift)))),

    interleavedmask := (b,idx,x1,x2) -> chain(
        assign(nth(tcast(TPtr(TSym("short int")), b), 2*idx),   interleavedmasklo_16x8i(x1,x2)),
        assign(nth(tcast(TPtr(TSym("short int")), b), 2*idx+1), interleavedmaskhi_16x8i(x1,x2))),

    average := (x1,x2) -> average_16x8i(x1,x2),

    #splats the first slot of the vector across the full vector
    dupload := (self, y, x) >> assign(y, vdup(x, self.v)),
    # computes the horizontal minimum of the vector and splats it
    hmin := (self,y,x,opts) >> let(
        sh_t := TVect(T_Int(64), 2),
	z := var.fresh_t("m", self.t),
	decl([z], chain(
	    assign(z, min(vec_shr(x, 8), x)),
	    assign(z, min(vec_shr(z, 4), z)),
            assign(z, min(vec_shr(z, 2), z)),
            assign(z, min(vec_shr(z, 1), z)),
	    assign(z, vunpacklo_16x8i(z, z)),
	    assign(z, vushufflelo_8x16i(z, [1,1,1,1])),
	    assign(y, vunpacklo_2x64i(z, z))))
    ),

    # keep the n lower scalars and zero the other ones
    optional_mask :=  (c, n, opts) -> When(IsBound(opts.trueSVSemantics) and opts.trueSVSemantics and not(n=16),
                          let(f:="0xFF", z:="0x0", bin_and(c,vhex(List([1..16],x->When(x<=n,f,z))))),
                          c),
    # load contiguous with unaligned loads
    loadc := (self, sv, opts) >> ((y,x) -> assign(y, self.optional_mask(vloadu_16x8i(x.toPtr(self.t.t)), sv, opts))),

    # Store contiguous unaligned
    storec_init := (vt) -> [
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        #(y,x) -> assign(y, vextract2_16x8i(x, 0)),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        #(y,x) -> assign(nth(y.toPtr(TVect(vt.t, 4)),0), vextract4_16x8i(x)),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        #(y,x) -> vstore8_16x8i(y.toPtr(TVect(vt.t, 8)), x),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0", "0x0"]),
        (y,x) -> vstoremsk_16x8i(y.toPtr(vt.t), x, ["0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0xFF", "0x0"]),
        (y,x) -> vstoreu_16x8i(y.toPtr(vt), x)
    ],

    storec := ~.storec_init(~.t),
));

SIMD_ISA_DB.addISA(SSE_2x64f);
SIMD_ISA_DB.addISA(SSE_2x64i);
SIMD_ISA_DB.addISA(SSE_2x32f);
SIMD_ISA_DB.addISA(SSE_4x32f);
SIMD_ISA_DB.addISA(SSE_4x32i);
SIMD_ISA_DB.addISA(SSE_8x16i);
SIMD_ISA_DB.addISA(SSE_16x8i);