gotosocial/vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s

// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.

//go:build !appengine && !noasm && gc && !noasm

#include "textflag.h"

// func _dummy_()
TEXT ·_dummy_(SB), $0
#ifdef GOAMD64_v4
#ifndef GOAMD64_v3
#define GOAMD64_v3
#endif
#endif
	RET

// func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ R9, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm
	LEAL  1(DX), DI
	MOVL  12(SP), R8
	MOVL  DI, SI
	SUBL  16(SP), SI
	JZ    repeat_extend_back_end_encodeBlockAsm

repeat_extend_back_loop_encodeBlockAsm:
	CMPL DI, R8
	JBE  repeat_extend_back_end_encodeBlockAsm
	MOVB -1(BX)(SI*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm
	LEAL -1(DI), DI
	DECL SI
	JNZ  repeat_extend_back_loop_encodeBlockAsm

repeat_extend_back_end_encodeBlockAsm:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 5(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBlockAsm:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeBlockAsm
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeBlockAsm
	CMPL SI, $0x01000000
	JB   four_bytes_repeat_emit_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm

four_bytes_repeat_emit_encodeBlockAsm:
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm

three_bytes_repeat_emit_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm

two_bytes_repeat_emit_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeBlockAsm
	JMP  memmove_long_repeat_emit_encodeBlockAsm

one_byte_repeat_emit_encodeBlockAsm:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm

emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm

emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_repeat_emit_encodeBlockAsm

emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_repeat_emit_encodeBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBlockAsm

memmove_long_repeat_emit_encodeBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R12
	SHRQ  $0x05, R12
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R13*1), R11
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R13*1), X4
	MOVOU -16(R10)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R9, R13
	JAE   emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBlockAsm:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R9
	SUBL DX, R9
	LEAQ (BX)(DX*1), R10
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R12, R12

matchlen_loopback_16_repeat_extend_encodeBlockAsm:
	CMPL R9, $0x10
	JB   matchlen_match8_repeat_extend_encodeBlockAsm
	MOVQ (R10)(R12*1), R11
	MOVQ 8(R10)(R12*1), R13
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm
	XORQ 8(SI)(R12*1), R13
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm
	LEAL -16(R9), R9
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_repeat_extend_encodeBlockAsm

matchlen_bsf_16repeat_extend_encodeBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm

matchlen_match8_repeat_extend_encodeBlockAsm:
	CMPL R9, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm
	MOVQ (R10)(R12*1), R11
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm
	LEAL -8(R9), R9
	LEAL 8(R12), R12
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm

matchlen_bsf_8_repeat_extend_encodeBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm

matchlen_match4_repeat_extend_encodeBlockAsm:
	CMPL R9, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm
	MOVL (R10)(R12*1), R11
	CMPL (SI)(R12*1), R11
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm
	LEAL -4(R9), R9
	LEAL 4(R12), R12

matchlen_match2_repeat_extend_encodeBlockAsm:
	CMPL R9, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm
	JB   repeat_extend_forward_end_encodeBlockAsm
	MOVW (R10)(R12*1), R11
	CMPW (SI)(R12*1), R11
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm
	LEAL 2(R12), R12
	SUBL $0x02, R9
	JZ   repeat_extend_forward_end_encodeBlockAsm

matchlen_match1_repeat_extend_encodeBlockAsm:
	MOVB (R10)(R12*1), R11
	CMPB (SI)(R12*1), R11
	JNE  repeat_extend_forward_end_encodeBlockAsm
	LEAL 1(R12), R12

repeat_extend_forward_end_encodeBlockAsm:
	ADDL  R12, DX
	MOVL  DX, SI
	SUBL  DI, SI
	MOVL  16(SP), DI
	TESTL R8, R8
	JZ    repeat_as_copy_encodeBlockAsm

	// emitRepeat
emit_repeat_again_match_repeat_encodeBlockAsm:
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_match_repeat_encodeBlockAsm
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_match_repeat_encodeBlockAsm
	CMPL DI, $0x00000800
	JB   repeat_two_offset_match_repeat_encodeBlockAsm

cant_repeat_two_offset_match_repeat_encodeBlockAsm:
	CMPL SI, $0x00000104
	JB   repeat_three_match_repeat_encodeBlockAsm
	CMPL SI, $0x00010100
	JB   repeat_four_match_repeat_encodeBlockAsm
	CMPL SI, $0x0100ffff
	JB   repeat_five_match_repeat_encodeBlockAsm
	LEAL -16842747(SI), SI
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_repeat_encodeBlockAsm

repeat_five_match_repeat_encodeBlockAsm:
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_four_match_repeat_encodeBlockAsm:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_three_match_repeat_encodeBlockAsm:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_match_repeat_encodeBlockAsm:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_offset_match_repeat_encodeBlockAsm:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_as_copy_encodeBlockAsm:
	// emitCopy
	CMPL DI, $0x00010000
	JB   two_byte_offset_repeat_as_copy_encodeBlockAsm
	CMPL SI, $0x40
	JBE  four_bytes_remain_repeat_as_copy_encodeBlockAsm
	MOVB $0xff, (CX)
	MOVL DI, 1(CX)
	LEAL -64(SI), SI
	ADDQ $0x05, CX
	CMPL SI, $0x04
	JB   four_bytes_remain_repeat_as_copy_encodeBlockAsm

	// emitRepeat
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy:
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy
	CMPL SI, $0x0100ffff
	JB   repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy
	LEAL -16842747(SI), SI
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy

repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy:
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

four_bytes_remain_repeat_as_copy_encodeBlockAsm:
	TESTL SI, SI
	JZ    repeat_end_emit_encodeBlockAsm
	XORL  R8, R8
	LEAL  -1(R8)(SI*4), SI
	MOVB  SI, (CX)
	MOVL  DI, 1(CX)
	ADDQ  $0x05, CX
	JMP   repeat_end_emit_encodeBlockAsm

two_byte_offset_repeat_as_copy_encodeBlockAsm:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeBlockAsm
	CMPL DI, $0x00000800
	JAE  long_offset_short_repeat_as_copy_encodeBlockAsm
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB DI, 1(CX)
	MOVL DI, R9
	SHRL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, SI

	// emitRepeat
	LEAL -4(SI), SI
	JMP  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b

emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
	CMPL SI, $0x0100ffff
	JB   repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b
	LEAL -16842747(SI), SI
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b

repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

long_offset_short_repeat_as_copy_encodeBlockAsm:
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX

	// emitRepeat
emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short
	CMPL SI, $0x0100ffff
	JB   repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short
	LEAL -16842747(SI), SI
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short

repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

two_byte_offset_short_repeat_as_copy_encodeBlockAsm:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm

emit_copy_three_repeat_as_copy_encodeBlockAsm:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeBlockAsm:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm

no_repeat_found_encodeBlockAsm:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm

candidate3_match_encodeBlockAsm:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm

candidate2_match_encodeBlockAsm:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm

match_extend_back_loop_encodeBlockAsm:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm
	JMP  match_extend_back_loop_encodeBlockAsm

match_extend_back_end_encodeBlockAsm:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 5(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBlockAsm:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeBlockAsm
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm
	CMPL R8, $0x00010000
	JB   three_bytes_match_emit_encodeBlockAsm
	CMPL R8, $0x01000000
	JB   four_bytes_match_emit_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL R8, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_encodeBlockAsm

four_bytes_match_emit_encodeBlockAsm:
	MOVL R8, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R8, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeBlockAsm

three_bytes_match_emit_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBlockAsm

two_bytes_match_emit_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeBlockAsm
	JMP  memmove_long_match_emit_encodeBlockAsm

one_byte_match_emit_encodeBlockAsm:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBlockAsm:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm

emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBlockAsm:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeBlockAsm

memmove_long_match_emit_encodeBlockAsm:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeBlockAsm:
match_nolit_loop_encodeBlockAsm:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeBlockAsm:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeBlockAsm
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeBlockAsm

matchlen_bsf_16match_nolit_encodeBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeBlockAsm

matchlen_match8_match_nolit_encodeBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeBlockAsm

matchlen_bsf_8_match_nolit_encodeBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeBlockAsm

matchlen_match4_match_nolit_encodeBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm
	JB   match_nolit_end_encodeBlockAsm
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeBlockAsm
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm

matchlen_match1_match_nolit_encodeBlockAsm:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeBlockAsm
	LEAL 1(R10), R10

match_nolit_end_encodeBlockAsm:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL SI, $0x00010000
	JB   two_byte_offset_match_nolit_encodeBlockAsm
	CMPL R10, $0x40
	JBE  four_bytes_remain_match_nolit_encodeBlockAsm
	MOVB $0xff, (CX)
	MOVL SI, 1(CX)
	LEAL -64(R10), R10
	ADDQ $0x05, CX
	CMPL R10, $0x04
	JB   four_bytes_remain_match_nolit_encodeBlockAsm

	// emitRepeat
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy:
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm_emit_copy
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy

cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm_emit_copy
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm_emit_copy
	CMPL R10, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBlockAsm_emit_copy
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy

repeat_five_match_nolit_encodeBlockAsm_emit_copy:
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_four_match_nolit_encodeBlockAsm_emit_copy:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_three_match_nolit_encodeBlockAsm_emit_copy:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_match_nolit_encodeBlockAsm_emit_copy:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

four_bytes_remain_match_nolit_encodeBlockAsm:
	TESTL R10, R10
	JZ    match_nolit_emitcopy_end_encodeBlockAsm
	XORL  DI, DI
	LEAL  -1(DI)(R10*4), R10
	MOVB  R10, (CX)
	MOVL  SI, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeBlockAsm

two_byte_offset_match_nolit_encodeBlockAsm:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBlockAsm
	CMPL SI, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBlockAsm
	MOVL $0x00000001, DI
	LEAL 16(DI), DI
	MOVB SI, 1(CX)
	MOVL SI, R8
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b

emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b
	CMPL R10, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b

repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

long_offset_short_match_nolit_encodeBlockAsm:
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX

	// emitRepeat
emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short:
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm_emit_copy_short
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm_emit_copy_short
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm_emit_copy_short
	CMPL R10, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBlockAsm_emit_copy_short
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short

repeat_five_match_nolit_encodeBlockAsm_emit_copy_short:
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_four_match_nolit_encodeBlockAsm_emit_copy_short:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_three_match_nolit_encodeBlockAsm_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_match_nolit_encodeBlockAsm_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

two_byte_offset_short_match_nolit_encodeBlockAsm:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBlockAsm
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBlockAsm
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm

emit_copy_three_match_nolit_encodeBlockAsm:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm:
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, SI
	IMULQ R9, SI
	SHRQ  $0x32, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeBlockAsm
	INCL  DX
	JMP   search_loop_encodeBlockAsm

emit_remainder_encodeBlockAsm:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 5(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBlockAsm
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBlockAsm
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBlockAsm
	CMPL DX, $0x01000000
	JB   four_bytes_emit_remainder_encodeBlockAsm
	MOVB $0xfc, (CX)
	MOVL DX, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm

four_bytes_emit_remainder_encodeBlockAsm:
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm

three_bytes_emit_remainder_encodeBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm

two_bytes_emit_remainder_encodeBlockAsm:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBlockAsm
	JMP  memmove_long_emit_remainder_encodeBlockAsm

one_byte_emit_remainder_encodeBlockAsm:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm

emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBlockAsm

memmove_long_emit_remainder_encodeBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBlockAsm4MB(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm4MB:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm4MB
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm4MB:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm4MB
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ R9, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm4MB
	LEAL  1(DX), DI
	MOVL  12(SP), R8
	MOVL  DI, SI
	SUBL  16(SP), SI
	JZ    repeat_extend_back_end_encodeBlockAsm4MB

repeat_extend_back_loop_encodeBlockAsm4MB:
	CMPL DI, R8
	JBE  repeat_extend_back_end_encodeBlockAsm4MB
	MOVB -1(BX)(SI*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm4MB
	LEAL -1(DI), DI
	DECL SI
	JNZ  repeat_extend_back_loop_encodeBlockAsm4MB

repeat_extend_back_end_encodeBlockAsm4MB:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 4(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBlockAsm4MB:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeBlockAsm4MB
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeBlockAsm4MB
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBlockAsm4MB
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeBlockAsm4MB
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm4MB

three_bytes_repeat_emit_encodeBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm4MB

two_bytes_repeat_emit_encodeBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeBlockAsm4MB
	JMP  memmove_long_repeat_emit_encodeBlockAsm4MB

one_byte_repeat_emit_encodeBlockAsm4MB:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm4MB

emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm4MB

emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_repeat_emit_encodeBlockAsm4MB

emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_repeat_emit_encodeBlockAsm4MB:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBlockAsm4MB

memmove_long_repeat_emit_encodeBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R12
	SHRQ  $0x05, R12
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R13*1), R11
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R13*1), X4
	MOVOU -16(R10)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R9, R13
	JAE   emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBlockAsm4MB:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R9
	SUBL DX, R9
	LEAQ (BX)(DX*1), R10
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R12, R12

matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB:
	CMPL R9, $0x10
	JB   matchlen_match8_repeat_extend_encodeBlockAsm4MB
	MOVQ (R10)(R12*1), R11
	MOVQ 8(R10)(R12*1), R13
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
	XORQ 8(SI)(R12*1), R13
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm4MB
	LEAL -16(R9), R9
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB

matchlen_bsf_16repeat_extend_encodeBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm4MB

matchlen_match8_repeat_extend_encodeBlockAsm4MB:
	CMPL R9, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm4MB
	MOVQ (R10)(R12*1), R11
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB
	LEAL -8(R9), R9
	LEAL 8(R12), R12
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm4MB

matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm4MB

matchlen_match4_repeat_extend_encodeBlockAsm4MB:
	CMPL R9, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm4MB
	MOVL (R10)(R12*1), R11
	CMPL (SI)(R12*1), R11
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm4MB
	LEAL -4(R9), R9
	LEAL 4(R12), R12

matchlen_match2_repeat_extend_encodeBlockAsm4MB:
	CMPL R9, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm4MB
	JB   repeat_extend_forward_end_encodeBlockAsm4MB
	MOVW (R10)(R12*1), R11
	CMPW (SI)(R12*1), R11
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm4MB
	LEAL 2(R12), R12
	SUBL $0x02, R9
	JZ   repeat_extend_forward_end_encodeBlockAsm4MB

matchlen_match1_repeat_extend_encodeBlockAsm4MB:
	MOVB (R10)(R12*1), R11
	CMPB (SI)(R12*1), R11
	JNE  repeat_extend_forward_end_encodeBlockAsm4MB
	LEAL 1(R12), R12

repeat_extend_forward_end_encodeBlockAsm4MB:
	ADDL  R12, DX
	MOVL  DX, SI
	SUBL  DI, SI
	MOVL  16(SP), DI
	TESTL R8, R8
	JZ    repeat_as_copy_encodeBlockAsm4MB

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_match_repeat_encodeBlockAsm4MB
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB
	CMPL DI, $0x00000800
	JB   repeat_two_offset_match_repeat_encodeBlockAsm4MB

cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB:
	CMPL SI, $0x00000104
	JB   repeat_three_match_repeat_encodeBlockAsm4MB
	CMPL SI, $0x00010100
	JB   repeat_four_match_repeat_encodeBlockAsm4MB
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_four_match_repeat_encodeBlockAsm4MB:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_three_match_repeat_encodeBlockAsm4MB:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_match_repeat_encodeBlockAsm4MB:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_offset_match_repeat_encodeBlockAsm4MB:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_as_copy_encodeBlockAsm4MB:
	// emitCopy
	CMPL DI, $0x00010000
	JB   two_byte_offset_repeat_as_copy_encodeBlockAsm4MB
	CMPL SI, $0x40
	JBE  four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB
	MOVB $0xff, (CX)
	MOVL DI, 1(CX)
	LEAL -64(SI), SI
	ADDQ $0x05, CX
	CMPL SI, $0x04
	JB   four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB:
	TESTL SI, SI
	JZ    repeat_end_emit_encodeBlockAsm4MB
	XORL  R8, R8
	LEAL  -1(R8)(SI*4), SI
	MOVB  SI, (CX)
	MOVL  DI, 1(CX)
	ADDQ  $0x05, CX
	JMP   repeat_end_emit_encodeBlockAsm4MB

two_byte_offset_repeat_as_copy_encodeBlockAsm4MB:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB
	CMPL DI, $0x00000800
	JAE  long_offset_short_repeat_as_copy_encodeBlockAsm4MB
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, SI

	// emitRepeat
	LEAL -4(SI), SI
	JMP  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

long_offset_short_repeat_as_copy_encodeBlockAsm4MB:
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
	CMPL SI, $0x00010100
	JB   repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short
	LEAL -65536(SI), SI
	MOVL SI, DI
	MOVW $0x001d, (CX)
	MOVW SI, 2(CX)
	SARL $0x10, DI
	MOVB DI, 4(CX)
	ADDQ $0x05, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm4MB
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm4MB

emit_copy_three_repeat_as_copy_encodeBlockAsm4MB:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeBlockAsm4MB:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm4MB

no_repeat_found_encodeBlockAsm4MB:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm4MB
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm4MB
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm4MB
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm4MB

candidate3_match_encodeBlockAsm4MB:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm4MB

candidate2_match_encodeBlockAsm4MB:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm4MB:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm4MB

match_extend_back_loop_encodeBlockAsm4MB:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm4MB
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm4MB
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm4MB
	JMP  match_extend_back_loop_encodeBlockAsm4MB

match_extend_back_end_encodeBlockAsm4MB:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 4(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBlockAsm4MB:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeBlockAsm4MB
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeBlockAsm4MB
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm4MB
	CMPL R8, $0x00010000
	JB   three_bytes_match_emit_encodeBlockAsm4MB
	MOVL R8, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R8, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeBlockAsm4MB

three_bytes_match_emit_encodeBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBlockAsm4MB

two_bytes_match_emit_encodeBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeBlockAsm4MB
	JMP  memmove_long_match_emit_encodeBlockAsm4MB

one_byte_match_emit_encodeBlockAsm4MB:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBlockAsm4MB:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm4MB

emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm4MB

emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm4MB

emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBlockAsm4MB:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeBlockAsm4MB

memmove_long_match_emit_encodeBlockAsm4MB:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeBlockAsm4MB:
match_nolit_loop_encodeBlockAsm4MB:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeBlockAsm4MB:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeBlockAsm4MB
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm4MB
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeBlockAsm4MB

matchlen_bsf_16match_nolit_encodeBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeBlockAsm4MB

matchlen_match8_match_nolit_encodeBlockAsm4MB:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm4MB
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm4MB
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeBlockAsm4MB

matchlen_bsf_8_match_nolit_encodeBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeBlockAsm4MB

matchlen_match4_match_nolit_encodeBlockAsm4MB:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm4MB
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeBlockAsm4MB
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeBlockAsm4MB:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm4MB
	JB   match_nolit_end_encodeBlockAsm4MB
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeBlockAsm4MB
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm4MB

matchlen_match1_match_nolit_encodeBlockAsm4MB:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeBlockAsm4MB
	LEAL 1(R10), R10

match_nolit_end_encodeBlockAsm4MB:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL SI, $0x00010000
	JB   two_byte_offset_match_nolit_encodeBlockAsm4MB
	CMPL R10, $0x40
	JBE  four_bytes_remain_match_nolit_encodeBlockAsm4MB
	MOVB $0xff, (CX)
	MOVL SI, 1(CX)
	LEAL -64(R10), R10
	ADDQ $0x05, CX
	CMPL R10, $0x04
	JB   four_bytes_remain_match_nolit_encodeBlockAsm4MB

	// emitRepeat
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy

cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

four_bytes_remain_match_nolit_encodeBlockAsm4MB:
	TESTL R10, R10
	JZ    match_nolit_emitcopy_end_encodeBlockAsm4MB
	XORL  DI, DI
	LEAL  -1(DI)(R10*4), R10
	MOVB  R10, (CX)
	MOVL  SI, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeBlockAsm4MB

two_byte_offset_match_nolit_encodeBlockAsm4MB:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBlockAsm4MB
	CMPL SI, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBlockAsm4MB
	MOVL $0x00000001, DI
	LEAL 16(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

long_offset_short_match_nolit_encodeBlockAsm4MB:
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short
	CMPL R10, $0x00010100
	JB   repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short
	LEAL -65536(R10), R10
	MOVL R10, SI
	MOVW $0x001d, (CX)
	MOVW R10, 2(CX)
	SARL $0x10, SI
	MOVB SI, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short:
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

two_byte_offset_short_match_nolit_encodeBlockAsm4MB:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBlockAsm4MB
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBlockAsm4MB
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm4MB

emit_copy_three_match_nolit_encodeBlockAsm4MB:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm4MB:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm4MB
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm4MB:
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, SI
	IMULQ R9, SI
	SHRQ  $0x32, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeBlockAsm4MB
	INCL  DX
	JMP   search_loop_encodeBlockAsm4MB

emit_remainder_encodeBlockAsm4MB:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBlockAsm4MB:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBlockAsm4MB
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBlockAsm4MB
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm4MB
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBlockAsm4MB
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm4MB

three_bytes_emit_remainder_encodeBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm4MB

two_bytes_emit_remainder_encodeBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBlockAsm4MB
	JMP  memmove_long_emit_remainder_encodeBlockAsm4MB

one_byte_emit_remainder_encodeBlockAsm4MB:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBlockAsm4MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm4MB:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBlockAsm4MB

memmove_long_emit_remainder_encodeBlockAsm4MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBlockAsm4MB:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBlockAsm12B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000080, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm12B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm12B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm12B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm12B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x000000cf1bbcdcbb, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x18, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	SHLQ  $0x18, R11
	IMULQ R9, R11
	SHRQ  $0x34, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x18, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm12B
	LEAL  1(DX), DI
	MOVL  12(SP), R8
	MOVL  DI, SI
	SUBL  16(SP), SI
	JZ    repeat_extend_back_end_encodeBlockAsm12B

repeat_extend_back_loop_encodeBlockAsm12B:
	CMPL DI, R8
	JBE  repeat_extend_back_end_encodeBlockAsm12B
	MOVB -1(BX)(SI*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm12B
	LEAL -1(DI), DI
	DECL SI
	JNZ  repeat_extend_back_loop_encodeBlockAsm12B

repeat_extend_back_end_encodeBlockAsm12B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBlockAsm12B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeBlockAsm12B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBlockAsm12B
	JB   three_bytes_repeat_emit_encodeBlockAsm12B

three_bytes_repeat_emit_encodeBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm12B

two_bytes_repeat_emit_encodeBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeBlockAsm12B
	JMP  memmove_long_repeat_emit_encodeBlockAsm12B

one_byte_repeat_emit_encodeBlockAsm12B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm12B

emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm12B

emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_repeat_emit_encodeBlockAsm12B

emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_repeat_emit_encodeBlockAsm12B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBlockAsm12B

memmove_long_repeat_emit_encodeBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R12
	SHRQ  $0x05, R12
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R13*1), R11
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R13*1), X4
	MOVOU -16(R10)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R9, R13
	JAE   emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBlockAsm12B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R9
	SUBL DX, R9
	LEAQ (BX)(DX*1), R10
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R12, R12

matchlen_loopback_16_repeat_extend_encodeBlockAsm12B:
	CMPL R9, $0x10
	JB   matchlen_match8_repeat_extend_encodeBlockAsm12B
	MOVQ (R10)(R12*1), R11
	MOVQ 8(R10)(R12*1), R13
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
	XORQ 8(SI)(R12*1), R13
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm12B
	LEAL -16(R9), R9
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_repeat_extend_encodeBlockAsm12B

matchlen_bsf_16repeat_extend_encodeBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm12B

matchlen_match8_repeat_extend_encodeBlockAsm12B:
	CMPL R9, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm12B
	MOVQ (R10)(R12*1), R11
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm12B
	LEAL -8(R9), R9
	LEAL 8(R12), R12
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm12B

matchlen_bsf_8_repeat_extend_encodeBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm12B

matchlen_match4_repeat_extend_encodeBlockAsm12B:
	CMPL R9, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm12B
	MOVL (R10)(R12*1), R11
	CMPL (SI)(R12*1), R11
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm12B
	LEAL -4(R9), R9
	LEAL 4(R12), R12

matchlen_match2_repeat_extend_encodeBlockAsm12B:
	CMPL R9, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm12B
	JB   repeat_extend_forward_end_encodeBlockAsm12B
	MOVW (R10)(R12*1), R11
	CMPW (SI)(R12*1), R11
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm12B
	LEAL 2(R12), R12
	SUBL $0x02, R9
	JZ   repeat_extend_forward_end_encodeBlockAsm12B

matchlen_match1_repeat_extend_encodeBlockAsm12B:
	MOVB (R10)(R12*1), R11
	CMPB (SI)(R12*1), R11
	JNE  repeat_extend_forward_end_encodeBlockAsm12B
	LEAL 1(R12), R12

repeat_extend_forward_end_encodeBlockAsm12B:
	ADDL  R12, DX
	MOVL  DX, SI
	SUBL  DI, SI
	MOVL  16(SP), DI
	TESTL R8, R8
	JZ    repeat_as_copy_encodeBlockAsm12B

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_match_repeat_encodeBlockAsm12B
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_match_repeat_encodeBlockAsm12B
	CMPL DI, $0x00000800
	JB   repeat_two_offset_match_repeat_encodeBlockAsm12B

cant_repeat_two_offset_match_repeat_encodeBlockAsm12B:
	CMPL SI, $0x00000104
	JB   repeat_three_match_repeat_encodeBlockAsm12B
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_three_match_repeat_encodeBlockAsm12B:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_match_repeat_encodeBlockAsm12B:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_offset_match_repeat_encodeBlockAsm12B:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_as_copy_encodeBlockAsm12B:
	// emitCopy
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B
	CMPL DI, $0x00000800
	JAE  long_offset_short_repeat_as_copy_encodeBlockAsm12B
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, SI

	// emitRepeat
	LEAL -4(SI), SI
	JMP  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

long_offset_short_repeat_as_copy_encodeBlockAsm12B:
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm12B
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm12B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm12B

emit_copy_three_repeat_as_copy_encodeBlockAsm12B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeBlockAsm12B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm12B

no_repeat_found_encodeBlockAsm12B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm12B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm12B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm12B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm12B

candidate3_match_encodeBlockAsm12B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm12B

candidate2_match_encodeBlockAsm12B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm12B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm12B

match_extend_back_loop_encodeBlockAsm12B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm12B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm12B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm12B
	JMP  match_extend_back_loop_encodeBlockAsm12B

match_extend_back_end_encodeBlockAsm12B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBlockAsm12B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeBlockAsm12B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm12B
	JB   three_bytes_match_emit_encodeBlockAsm12B

three_bytes_match_emit_encodeBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBlockAsm12B

two_bytes_match_emit_encodeBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeBlockAsm12B
	JMP  memmove_long_match_emit_encodeBlockAsm12B

one_byte_match_emit_encodeBlockAsm12B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBlockAsm12B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm12B

emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm12B

emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm12B

emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBlockAsm12B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeBlockAsm12B

memmove_long_match_emit_encodeBlockAsm12B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeBlockAsm12B:
match_nolit_loop_encodeBlockAsm12B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeBlockAsm12B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeBlockAsm12B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm12B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm12B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeBlockAsm12B

matchlen_bsf_16match_nolit_encodeBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeBlockAsm12B

matchlen_match8_match_nolit_encodeBlockAsm12B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm12B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm12B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeBlockAsm12B

matchlen_bsf_8_match_nolit_encodeBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeBlockAsm12B

matchlen_match4_match_nolit_encodeBlockAsm12B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm12B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeBlockAsm12B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeBlockAsm12B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm12B
	JB   match_nolit_end_encodeBlockAsm12B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeBlockAsm12B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm12B

matchlen_match1_match_nolit_encodeBlockAsm12B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeBlockAsm12B
	LEAL 1(R10), R10

match_nolit_end_encodeBlockAsm12B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBlockAsm12B
	CMPL SI, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBlockAsm12B
	MOVL $0x00000001, DI
	LEAL 16(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

long_offset_short_match_nolit_encodeBlockAsm12B:
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

two_byte_offset_short_match_nolit_encodeBlockAsm12B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBlockAsm12B
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBlockAsm12B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm12B

emit_copy_three_match_nolit_encodeBlockAsm12B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm12B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm12B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm12B:
	MOVQ  $0x000000cf1bbcdcbb, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x18, R8
	IMULQ R9, R8
	SHRQ  $0x34, R8
	SHLQ  $0x18, SI
	IMULQ R9, SI
	SHRQ  $0x34, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeBlockAsm12B
	INCL  DX
	JMP   search_loop_encodeBlockAsm12B

emit_remainder_encodeBlockAsm12B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBlockAsm12B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBlockAsm12B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBlockAsm12B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm12B
	JB   three_bytes_emit_remainder_encodeBlockAsm12B

three_bytes_emit_remainder_encodeBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm12B

two_bytes_emit_remainder_encodeBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBlockAsm12B
	JMP  memmove_long_emit_remainder_encodeBlockAsm12B

one_byte_emit_remainder_encodeBlockAsm12B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm12B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBlockAsm12B

memmove_long_emit_remainder_encodeBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBlockAsm12B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBlockAsm10B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000020, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm10B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm10B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm10B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm10B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ R9, R11
	SHRQ  $0x36, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm10B
	LEAL  1(DX), DI
	MOVL  12(SP), R8
	MOVL  DI, SI
	SUBL  16(SP), SI
	JZ    repeat_extend_back_end_encodeBlockAsm10B

repeat_extend_back_loop_encodeBlockAsm10B:
	CMPL DI, R8
	JBE  repeat_extend_back_end_encodeBlockAsm10B
	MOVB -1(BX)(SI*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm10B
	LEAL -1(DI), DI
	DECL SI
	JNZ  repeat_extend_back_loop_encodeBlockAsm10B

repeat_extend_back_end_encodeBlockAsm10B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBlockAsm10B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeBlockAsm10B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBlockAsm10B
	JB   three_bytes_repeat_emit_encodeBlockAsm10B

three_bytes_repeat_emit_encodeBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm10B

two_bytes_repeat_emit_encodeBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeBlockAsm10B
	JMP  memmove_long_repeat_emit_encodeBlockAsm10B

one_byte_repeat_emit_encodeBlockAsm10B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm10B

emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm10B

emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_repeat_emit_encodeBlockAsm10B

emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_repeat_emit_encodeBlockAsm10B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBlockAsm10B

memmove_long_repeat_emit_encodeBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R12
	SHRQ  $0x05, R12
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R13*1), R11
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R13*1), X4
	MOVOU -16(R10)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R9, R13
	JAE   emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBlockAsm10B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R9
	SUBL DX, R9
	LEAQ (BX)(DX*1), R10
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R12, R12

matchlen_loopback_16_repeat_extend_encodeBlockAsm10B:
	CMPL R9, $0x10
	JB   matchlen_match8_repeat_extend_encodeBlockAsm10B
	MOVQ (R10)(R12*1), R11
	MOVQ 8(R10)(R12*1), R13
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
	XORQ 8(SI)(R12*1), R13
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm10B
	LEAL -16(R9), R9
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_repeat_extend_encodeBlockAsm10B

matchlen_bsf_16repeat_extend_encodeBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm10B

matchlen_match8_repeat_extend_encodeBlockAsm10B:
	CMPL R9, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm10B
	MOVQ (R10)(R12*1), R11
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm10B
	LEAL -8(R9), R9
	LEAL 8(R12), R12
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm10B

matchlen_bsf_8_repeat_extend_encodeBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm10B

matchlen_match4_repeat_extend_encodeBlockAsm10B:
	CMPL R9, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm10B
	MOVL (R10)(R12*1), R11
	CMPL (SI)(R12*1), R11
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm10B
	LEAL -4(R9), R9
	LEAL 4(R12), R12

matchlen_match2_repeat_extend_encodeBlockAsm10B:
	CMPL R9, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm10B
	JB   repeat_extend_forward_end_encodeBlockAsm10B
	MOVW (R10)(R12*1), R11
	CMPW (SI)(R12*1), R11
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm10B
	LEAL 2(R12), R12
	SUBL $0x02, R9
	JZ   repeat_extend_forward_end_encodeBlockAsm10B

matchlen_match1_repeat_extend_encodeBlockAsm10B:
	MOVB (R10)(R12*1), R11
	CMPB (SI)(R12*1), R11
	JNE  repeat_extend_forward_end_encodeBlockAsm10B
	LEAL 1(R12), R12

repeat_extend_forward_end_encodeBlockAsm10B:
	ADDL  R12, DX
	MOVL  DX, SI
	SUBL  DI, SI
	MOVL  16(SP), DI
	TESTL R8, R8
	JZ    repeat_as_copy_encodeBlockAsm10B

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_match_repeat_encodeBlockAsm10B
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_match_repeat_encodeBlockAsm10B
	CMPL DI, $0x00000800
	JB   repeat_two_offset_match_repeat_encodeBlockAsm10B

cant_repeat_two_offset_match_repeat_encodeBlockAsm10B:
	CMPL SI, $0x00000104
	JB   repeat_three_match_repeat_encodeBlockAsm10B
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_three_match_repeat_encodeBlockAsm10B:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_match_repeat_encodeBlockAsm10B:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_offset_match_repeat_encodeBlockAsm10B:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_as_copy_encodeBlockAsm10B:
	// emitCopy
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B
	CMPL DI, $0x00000800
	JAE  long_offset_short_repeat_as_copy_encodeBlockAsm10B
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, SI

	// emitRepeat
	LEAL -4(SI), SI
	JMP  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

long_offset_short_repeat_as_copy_encodeBlockAsm10B:
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX

	// emitRepeat
	MOVL SI, R8
	LEAL -4(SI), SI
	CMPL R8, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
	CMPL DI, $0x00000800
	JB   repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm10B
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm10B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm10B

emit_copy_three_repeat_as_copy_encodeBlockAsm10B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeBlockAsm10B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm10B

no_repeat_found_encodeBlockAsm10B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm10B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm10B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm10B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm10B

candidate3_match_encodeBlockAsm10B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm10B

candidate2_match_encodeBlockAsm10B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm10B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm10B

match_extend_back_loop_encodeBlockAsm10B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm10B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm10B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm10B
	JMP  match_extend_back_loop_encodeBlockAsm10B

match_extend_back_end_encodeBlockAsm10B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBlockAsm10B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeBlockAsm10B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm10B
	JB   three_bytes_match_emit_encodeBlockAsm10B

three_bytes_match_emit_encodeBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBlockAsm10B

two_bytes_match_emit_encodeBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeBlockAsm10B
	JMP  memmove_long_match_emit_encodeBlockAsm10B

one_byte_match_emit_encodeBlockAsm10B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBlockAsm10B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm10B

emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm10B

emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm10B

emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBlockAsm10B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeBlockAsm10B

memmove_long_match_emit_encodeBlockAsm10B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeBlockAsm10B:
match_nolit_loop_encodeBlockAsm10B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeBlockAsm10B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeBlockAsm10B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm10B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm10B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeBlockAsm10B

matchlen_bsf_16match_nolit_encodeBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeBlockAsm10B

matchlen_match8_match_nolit_encodeBlockAsm10B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm10B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm10B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeBlockAsm10B

matchlen_bsf_8_match_nolit_encodeBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeBlockAsm10B

matchlen_match4_match_nolit_encodeBlockAsm10B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm10B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeBlockAsm10B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeBlockAsm10B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm10B
	JB   match_nolit_end_encodeBlockAsm10B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeBlockAsm10B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm10B

matchlen_match1_match_nolit_encodeBlockAsm10B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeBlockAsm10B
	LEAL 1(R10), R10

match_nolit_end_encodeBlockAsm10B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBlockAsm10B
	CMPL SI, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBlockAsm10B
	MOVL $0x00000001, DI
	LEAL 16(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

long_offset_short_match_nolit_encodeBlockAsm10B:
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R10, DI
	LEAL -4(R10), R10
	CMPL DI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short
	CMPL SI, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short:
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

two_byte_offset_short_match_nolit_encodeBlockAsm10B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBlockAsm10B
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBlockAsm10B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm10B

emit_copy_three_match_nolit_encodeBlockAsm10B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm10B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm10B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm10B:
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x20, R8
	IMULQ R9, R8
	SHRQ  $0x36, R8
	SHLQ  $0x20, SI
	IMULQ R9, SI
	SHRQ  $0x36, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeBlockAsm10B
	INCL  DX
	JMP   search_loop_encodeBlockAsm10B

emit_remainder_encodeBlockAsm10B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBlockAsm10B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBlockAsm10B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBlockAsm10B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm10B
	JB   three_bytes_emit_remainder_encodeBlockAsm10B

three_bytes_emit_remainder_encodeBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm10B

two_bytes_emit_remainder_encodeBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBlockAsm10B
	JMP  memmove_long_emit_remainder_encodeBlockAsm10B

one_byte_emit_remainder_encodeBlockAsm10B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm10B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBlockAsm10B

memmove_long_emit_remainder_encodeBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBlockAsm10B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBlockAsm8B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000008, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBlockAsm8B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBlockAsm8B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBlockAsm8B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x04, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBlockAsm8B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x38, R10
	SHLQ  $0x20, R11
	IMULQ R9, R11
	SHRQ  $0x38, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x38, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeBlockAsm8B
	LEAL  1(DX), DI
	MOVL  12(SP), R8
	MOVL  DI, SI
	SUBL  16(SP), SI
	JZ    repeat_extend_back_end_encodeBlockAsm8B

repeat_extend_back_loop_encodeBlockAsm8B:
	CMPL DI, R8
	JBE  repeat_extend_back_end_encodeBlockAsm8B
	MOVB -1(BX)(SI*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeBlockAsm8B
	LEAL -1(DI), DI
	DECL SI
	JNZ  repeat_extend_back_loop_encodeBlockAsm8B

repeat_extend_back_end_encodeBlockAsm8B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeBlockAsm8B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeBlockAsm8B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeBlockAsm8B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeBlockAsm8B
	JB   three_bytes_repeat_emit_encodeBlockAsm8B

three_bytes_repeat_emit_encodeBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeBlockAsm8B

two_bytes_repeat_emit_encodeBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeBlockAsm8B
	JMP  memmove_long_repeat_emit_encodeBlockAsm8B

one_byte_repeat_emit_encodeBlockAsm8B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm8B

emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_repeat_emit_encodeBlockAsm8B

emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_repeat_emit_encodeBlockAsm8B

emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_repeat_emit_encodeBlockAsm8B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeBlockAsm8B

memmove_long_repeat_emit_encodeBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R12
	SHRQ  $0x05, R12
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R13*1), R11
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R13*1), X4
	MOVOU -16(R10)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R9, R13
	JAE   emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeBlockAsm8B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R9
	SUBL DX, R9
	LEAQ (BX)(DX*1), R10
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R12, R12

matchlen_loopback_16_repeat_extend_encodeBlockAsm8B:
	CMPL R9, $0x10
	JB   matchlen_match8_repeat_extend_encodeBlockAsm8B
	MOVQ (R10)(R12*1), R11
	MOVQ 8(R10)(R12*1), R13
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
	XORQ 8(SI)(R12*1), R13
	JNZ  matchlen_bsf_16repeat_extend_encodeBlockAsm8B
	LEAL -16(R9), R9
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_repeat_extend_encodeBlockAsm8B

matchlen_bsf_16repeat_extend_encodeBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm8B

matchlen_match8_repeat_extend_encodeBlockAsm8B:
	CMPL R9, $0x08
	JB   matchlen_match4_repeat_extend_encodeBlockAsm8B
	MOVQ (R10)(R12*1), R11
	XORQ (SI)(R12*1), R11
	JNZ  matchlen_bsf_8_repeat_extend_encodeBlockAsm8B
	LEAL -8(R9), R9
	LEAL 8(R12), R12
	JMP  matchlen_match4_repeat_extend_encodeBlockAsm8B

matchlen_bsf_8_repeat_extend_encodeBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  repeat_extend_forward_end_encodeBlockAsm8B

matchlen_match4_repeat_extend_encodeBlockAsm8B:
	CMPL R9, $0x04
	JB   matchlen_match2_repeat_extend_encodeBlockAsm8B
	MOVL (R10)(R12*1), R11
	CMPL (SI)(R12*1), R11
	JNE  matchlen_match2_repeat_extend_encodeBlockAsm8B
	LEAL -4(R9), R9
	LEAL 4(R12), R12

matchlen_match2_repeat_extend_encodeBlockAsm8B:
	CMPL R9, $0x01
	JE   matchlen_match1_repeat_extend_encodeBlockAsm8B
	JB   repeat_extend_forward_end_encodeBlockAsm8B
	MOVW (R10)(R12*1), R11
	CMPW (SI)(R12*1), R11
	JNE  matchlen_match1_repeat_extend_encodeBlockAsm8B
	LEAL 2(R12), R12
	SUBL $0x02, R9
	JZ   repeat_extend_forward_end_encodeBlockAsm8B

matchlen_match1_repeat_extend_encodeBlockAsm8B:
	MOVB (R10)(R12*1), R11
	CMPB (SI)(R12*1), R11
	JNE  repeat_extend_forward_end_encodeBlockAsm8B
	LEAL 1(R12), R12

repeat_extend_forward_end_encodeBlockAsm8B:
	ADDL  R12, DX
	MOVL  DX, SI
	SUBL  DI, SI
	MOVL  16(SP), DI
	TESTL R8, R8
	JZ    repeat_as_copy_encodeBlockAsm8B

	// emitRepeat
	MOVL SI, DI
	LEAL -4(SI), SI
	CMPL DI, $0x08
	JBE  repeat_two_match_repeat_encodeBlockAsm8B
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_match_repeat_encodeBlockAsm8B

cant_repeat_two_offset_match_repeat_encodeBlockAsm8B:
	CMPL SI, $0x00000104
	JB   repeat_three_match_repeat_encodeBlockAsm8B
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_three_match_repeat_encodeBlockAsm8B:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_two_match_repeat_encodeBlockAsm8B:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_as_copy_encodeBlockAsm8B:
	// emitCopy
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B
	CMPL DI, $0x00000800
	JAE  long_offset_short_repeat_as_copy_encodeBlockAsm8B
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, SI

	// emitRepeat
	LEAL -4(SI), SI
	JMP  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
	MOVL SI, DI
	LEAL -4(SI), SI
	CMPL DI, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

long_offset_short_repeat_as_copy_encodeBlockAsm8B:
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX

	// emitRepeat
	MOVL SI, DI
	LEAL -4(SI), SI
	CMPL DI, $0x08
	JBE  repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
	CMPL DI, $0x0c
	JAE  cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short

cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
	CMPL SI, $0x00000104
	JB   repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short
	LEAL -256(SI), SI
	MOVW $0x0019, (CX)
	MOVW SI, 2(CX)
	ADDQ $0x04, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
	LEAL -4(SI), SI
	MOVW $0x0015, (CX)
	MOVB SI, 2(CX)
	ADDQ $0x03, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short:
	SHLL $0x02, SI
	ORL  $0x01, SI
	MOVW SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B
	XORQ R8, R8
	LEAL 1(R8)(SI*4), SI
	MOVB DI, 1(CX)
	SARL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeBlockAsm8B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeBlockAsm8B

emit_copy_three_repeat_as_copy_encodeBlockAsm8B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeBlockAsm8B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeBlockAsm8B

no_repeat_found_encodeBlockAsm8B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeBlockAsm8B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeBlockAsm8B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeBlockAsm8B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBlockAsm8B

candidate3_match_encodeBlockAsm8B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeBlockAsm8B

candidate2_match_encodeBlockAsm8B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeBlockAsm8B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBlockAsm8B

match_extend_back_loop_encodeBlockAsm8B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBlockAsm8B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBlockAsm8B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBlockAsm8B
	JMP  match_extend_back_loop_encodeBlockAsm8B

match_extend_back_end_encodeBlockAsm8B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBlockAsm8B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeBlockAsm8B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeBlockAsm8B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeBlockAsm8B
	JB   three_bytes_match_emit_encodeBlockAsm8B

three_bytes_match_emit_encodeBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBlockAsm8B

two_bytes_match_emit_encodeBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeBlockAsm8B
	JMP  memmove_long_match_emit_encodeBlockAsm8B

one_byte_match_emit_encodeBlockAsm8B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBlockAsm8B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm8B

emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBlockAsm8B

emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBlockAsm8B

emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBlockAsm8B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeBlockAsm8B

memmove_long_match_emit_encodeBlockAsm8B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeBlockAsm8B:
match_nolit_loop_encodeBlockAsm8B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeBlockAsm8B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeBlockAsm8B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm8B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeBlockAsm8B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeBlockAsm8B

matchlen_bsf_16match_nolit_encodeBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeBlockAsm8B

matchlen_match8_match_nolit_encodeBlockAsm8B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeBlockAsm8B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeBlockAsm8B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeBlockAsm8B

matchlen_bsf_8_match_nolit_encodeBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeBlockAsm8B

matchlen_match4_match_nolit_encodeBlockAsm8B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeBlockAsm8B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeBlockAsm8B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeBlockAsm8B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeBlockAsm8B
	JB   match_nolit_end_encodeBlockAsm8B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeBlockAsm8B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeBlockAsm8B

matchlen_match1_match_nolit_encodeBlockAsm8B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeBlockAsm8B
	LEAL 1(R10), R10

match_nolit_end_encodeBlockAsm8B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBlockAsm8B
	CMPL SI, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBlockAsm8B
	MOVL $0x00000001, DI
	LEAL 16(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
	MOVL R10, SI
	LEAL -4(R10), R10
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

long_offset_short_match_nolit_encodeBlockAsm8B:
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R10, SI
	LEAL -4(R10), R10
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short
	LEAL -256(R10), R10
	MOVW $0x0019, (CX)
	MOVW R10, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (CX)
	MOVB R10, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B
	XORQ DI, DI
	LEAL 1(DI)(R10*4), R10
	MOVB SI, 1(CX)
	SARL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, R10
	MOVB R10, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

two_byte_offset_short_match_nolit_encodeBlockAsm8B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBlockAsm8B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBlockAsm8B

emit_copy_three_match_nolit_encodeBlockAsm8B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeBlockAsm8B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBlockAsm8B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBlockAsm8B:
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x20, R8
	IMULQ R9, R8
	SHRQ  $0x38, R8
	SHLQ  $0x20, SI
	IMULQ R9, SI
	SHRQ  $0x38, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeBlockAsm8B
	INCL  DX
	JMP   search_loop_encodeBlockAsm8B

emit_remainder_encodeBlockAsm8B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBlockAsm8B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBlockAsm8B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBlockAsm8B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBlockAsm8B
	JB   three_bytes_emit_remainder_encodeBlockAsm8B

three_bytes_emit_remainder_encodeBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBlockAsm8B

two_bytes_emit_remainder_encodeBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBlockAsm8B
	JMP  memmove_long_emit_remainder_encodeBlockAsm8B

one_byte_emit_remainder_encodeBlockAsm8B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBlockAsm8B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBlockAsm8B

memmove_long_emit_remainder_encodeBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBlockAsm8B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBetterBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00001200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBetterBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -6(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBetterBlockAsm:
	MOVL DX, SI
	SUBL 12(SP), SI
	SHRL $0x07, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeBetterBlockAsm
	LEAL 100(DX), SI
	JMP  check_maxskip_cont_encodeBetterBlockAsm

check_maxskip_ok_encodeBetterBlockAsm:
	LEAL 1(DX)(SI*1), SI

check_maxskip_cont_encodeBetterBlockAsm:
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  524288(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 524288(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm
	CMPQ  R11, DI
	JNE   no_short_found_encodeBetterBlockAsm
	MOVL  R8, SI
	JMP   candidate_match_encodeBetterBlockAsm

no_short_found_encodeBetterBlockAsm:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm
	MOVL 20(SP), DX
	JMP  search_loop_encodeBetterBlockAsm

candidateS_match_encodeBetterBlockAsm:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeBetterBlockAsm
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeBetterBlockAsm:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm

match_extend_back_loop_encodeBetterBlockAsm:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBetterBlockAsm
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBetterBlockAsm
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm
	JMP  match_extend_back_loop_encodeBetterBlockAsm

match_extend_back_end_encodeBetterBlockAsm:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 5(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeBetterBlockAsm
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm

matchlen_bsf_16match_nolit_encodeBetterBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm

matchlen_match8_match_nolit_encodeBetterBlockAsm:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm

matchlen_match4_match_nolit_encodeBetterBlockAsm:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeBetterBlockAsm:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm
	JB   match_nolit_end_encodeBetterBlockAsm
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeBetterBlockAsm

matchlen_match1_match_nolit_encodeBetterBlockAsm:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeBetterBlockAsm
	LEAL 1(R12), R12

match_nolit_end_encodeBetterBlockAsm:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL 16(SP), R8
	JEQ  match_is_repeat_encodeBetterBlockAsm
	CMPL R12, $0x01
	JA   match_length_ok_encodeBetterBlockAsm
	CMPL R8, $0x0000ffff
	JBE  match_length_ok_encodeBetterBlockAsm
	MOVL 20(SP), DX
	INCL DX
	JMP  search_loop_encodeBetterBlockAsm

match_length_ok_encodeBetterBlockAsm:
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeBetterBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeBetterBlockAsm
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_encodeBetterBlockAsm
	CMPL SI, $0x01000000
	JB   four_bytes_match_emit_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

four_bytes_match_emit_encodeBetterBlockAsm:
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

three_bytes_match_emit_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

two_bytes_match_emit_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeBetterBlockAsm
	JMP  memmove_long_match_emit_encodeBetterBlockAsm

one_byte_match_emit_encodeBetterBlockAsm:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm

emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeBetterBlockAsm

memmove_long_match_emit_encodeBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeBetterBlockAsm:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R8, $0x00010000
	JB   two_byte_offset_match_nolit_encodeBetterBlockAsm
	CMPL R12, $0x40
	JBE  four_bytes_remain_match_nolit_encodeBetterBlockAsm
	MOVB $0xff, (CX)
	MOVL R8, 1(CX)
	LEAL -64(R12), R12
	ADDQ $0x05, CX
	CMPL R12, $0x04
	JB   four_bytes_remain_match_nolit_encodeBetterBlockAsm

	// emitRepeat
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy:
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy
	CMPL R12, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy
	LEAL -16842747(R12), R12
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy

repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy:
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

four_bytes_remain_match_nolit_encodeBetterBlockAsm:
	TESTL R12, R12
	JZ    match_nolit_emitcopy_end_encodeBetterBlockAsm
	XORL  SI, SI
	LEAL  -1(SI)(R12*4), R12
	MOVB  R12, (CX)
	MOVL  R8, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeBetterBlockAsm

two_byte_offset_match_nolit_encodeBetterBlockAsm:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBetterBlockAsm
	CMPL R8, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBetterBlockAsm
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB R8, 1(CX)
	MOVL R8, R9
	SHRL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R12

	// emitRepeat
	LEAL -4(R12), R12
	JMP  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b

emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
	CMPL R12, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b
	LEAL -16842747(R12), R12
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b

repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

long_offset_short_match_nolit_encodeBetterBlockAsm:
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX

	// emitRepeat
emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short
	CMPL R12, $0x0100ffff
	JB   repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short
	LEAL -16842747(R12), R12
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short

repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

two_byte_offset_short_match_nolit_encodeBetterBlockAsm:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

emit_copy_three_match_nolit_encodeBetterBlockAsm:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

match_is_repeat_encodeBetterBlockAsm:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm
	CMPL SI, $0x01000000
	JB   four_bytes_match_emit_repeat_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

four_bytes_match_emit_repeat_encodeBetterBlockAsm:
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

three_bytes_match_emit_repeat_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

two_bytes_match_emit_repeat_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_repeat_encodeBetterBlockAsm
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm

one_byte_match_emit_repeat_encodeBetterBlockAsm:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_repeat_encodeBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm

memmove_long_match_emit_repeat_encodeBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitRepeat
emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm:
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_repeat_encodeBetterBlockAsm
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm

cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_repeat_encodeBetterBlockAsm
	CMPL R12, $0x0100ffff
	JB   repeat_five_match_nolit_repeat_encodeBetterBlockAsm
	LEAL -16842747(R12), R12
	MOVL $0xfffb001d, (CX)
	MOVB $0xff, 4(CX)
	ADDQ $0x05, CX
	JMP  emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm

repeat_five_match_nolit_repeat_encodeBetterBlockAsm:
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_four_match_nolit_repeat_encodeBetterBlockAsm:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_three_match_nolit_repeat_encodeBetterBlockAsm:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_match_nolit_repeat_encodeBetterBlockAsm:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm

repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm:
	MOVQ  $0x00cf1bbcdcbfa563, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x32, R11
	SHLQ  $0x08, R12
	IMULQ SI, R12
	SHRQ  $0x2f, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x32, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 524288(AX)(R11*4)
	MOVL  R14, 524288(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeBetterBlockAsm:
	CMPQ  R8, R9
	JAE   search_loop_encodeBetterBlockAsm
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x08, R11
	IMULQ SI, R11
	SHRQ  $0x2f, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeBetterBlockAsm

emit_remainder_encodeBetterBlockAsm:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 5(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBetterBlockAsm
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm
	CMPL DX, $0x01000000
	JB   four_bytes_emit_remainder_encodeBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL DX, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

four_bytes_emit_remainder_encodeBetterBlockAsm:
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

three_bytes_emit_remainder_encodeBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

two_bytes_emit_remainder_encodeBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBetterBlockAsm
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm

one_byte_emit_remainder_encodeBetterBlockAsm:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm

memmove_long_emit_remainder_encodeBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBetterBlockAsm4MB(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00001200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm4MB:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBetterBlockAsm4MB
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -6(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBetterBlockAsm4MB:
	MOVL DX, SI
	SUBL 12(SP), SI
	SHRL $0x07, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeBetterBlockAsm4MB
	LEAL 100(DX), SI
	JMP  check_maxskip_cont_encodeBetterBlockAsm4MB

check_maxskip_ok_encodeBetterBlockAsm4MB:
	LEAL 1(DX)(SI*1), SI

check_maxskip_cont_encodeBetterBlockAsm4MB:
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm4MB
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  524288(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 524288(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm4MB
	CMPQ  R11, DI
	JNE   no_short_found_encodeBetterBlockAsm4MB
	MOVL  R8, SI
	JMP   candidate_match_encodeBetterBlockAsm4MB

no_short_found_encodeBetterBlockAsm4MB:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm4MB
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm4MB
	MOVL 20(SP), DX
	JMP  search_loop_encodeBetterBlockAsm4MB

candidateS_match_encodeBetterBlockAsm4MB:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeBetterBlockAsm4MB
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeBetterBlockAsm4MB:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm4MB

match_extend_back_loop_encodeBetterBlockAsm4MB:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBetterBlockAsm4MB
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBetterBlockAsm4MB
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm4MB
	JMP  match_extend_back_loop_encodeBetterBlockAsm4MB

match_extend_back_end_encodeBetterBlockAsm4MB:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 4(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm4MB:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeBetterBlockAsm4MB
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB

matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm4MB

matchlen_match8_match_nolit_encodeBetterBlockAsm4MB:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm4MB
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm4MB

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm4MB

matchlen_match4_match_nolit_encodeBetterBlockAsm4MB:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm4MB
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeBetterBlockAsm4MB:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
	JB   match_nolit_end_encodeBetterBlockAsm4MB
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm4MB
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeBetterBlockAsm4MB

matchlen_match1_match_nolit_encodeBetterBlockAsm4MB:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeBetterBlockAsm4MB
	LEAL 1(R12), R12

match_nolit_end_encodeBetterBlockAsm4MB:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL 16(SP), R8
	JEQ  match_is_repeat_encodeBetterBlockAsm4MB
	CMPL R12, $0x01
	JA   match_length_ok_encodeBetterBlockAsm4MB
	CMPL R8, $0x0000ffff
	JBE  match_length_ok_encodeBetterBlockAsm4MB
	MOVL 20(SP), DX
	INCL DX
	JMP  search_loop_encodeBetterBlockAsm4MB

match_length_ok_encodeBetterBlockAsm4MB:
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeBetterBlockAsm4MB
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeBetterBlockAsm4MB
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm4MB
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_encodeBetterBlockAsm4MB
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm4MB

three_bytes_match_emit_encodeBetterBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm4MB

two_bytes_match_emit_encodeBetterBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeBetterBlockAsm4MB
	JMP  memmove_long_match_emit_encodeBetterBlockAsm4MB

one_byte_match_emit_encodeBetterBlockAsm4MB:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBetterBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm4MB:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeBetterBlockAsm4MB

memmove_long_match_emit_encodeBetterBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeBetterBlockAsm4MB:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R8, $0x00010000
	JB   two_byte_offset_match_nolit_encodeBetterBlockAsm4MB
	CMPL R12, $0x40
	JBE  four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB
	MOVB $0xff, (CX)
	MOVL R8, 1(CX)
	LEAL -64(R12), R12
	ADDQ $0x05, CX
	CMPL R12, $0x04
	JB   four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB:
	TESTL R12, R12
	JZ    match_nolit_emitcopy_end_encodeBetterBlockAsm4MB
	XORL  SI, SI
	LEAL  -1(SI)(R12*4), R12
	MOVB  R12, (CX)
	MOVL  R8, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

two_byte_offset_match_nolit_encodeBetterBlockAsm4MB:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB
	CMPL R8, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBetterBlockAsm4MB
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R12

	// emitRepeat
	LEAL -4(R12), R12
	JMP  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

long_offset_short_match_nolit_encodeBetterBlockAsm4MB:
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm4MB
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

emit_copy_three_match_nolit_encodeBetterBlockAsm4MB:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

match_is_repeat_encodeBetterBlockAsm4MB:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm4MB
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB

three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB

two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_repeat_encodeBetterBlockAsm4MB
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB

one_byte_match_emit_repeat_encodeBetterBlockAsm4MB:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_repeat_encodeBetterBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB

memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB

cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB
	CMPL R12, $0x00010100
	JB   repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB
	LEAL -65536(R12), R12
	MOVL R12, R8
	MOVW $0x001d, (CX)
	MOVW R12, 2(CX)
	SARL $0x10, R8
	MOVB R8, 4(CX)
	ADDQ $0x05, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB:
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm4MB

repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm4MB:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm4MB
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm4MB:
	MOVQ  $0x00cf1bbcdcbfa563, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x32, R11
	SHLQ  $0x08, R12
	IMULQ SI, R12
	SHRQ  $0x2f, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x32, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 524288(AX)(R11*4)
	MOVL  R14, 524288(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeBetterBlockAsm4MB:
	CMPQ  R8, R9
	JAE   search_loop_encodeBetterBlockAsm4MB
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x08, R11
	IMULQ SI, R11
	SHRQ  $0x2f, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeBetterBlockAsm4MB

emit_remainder_encodeBetterBlockAsm4MB:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 4(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm4MB
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm4MB:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBetterBlockAsm4MB
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm4MB
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm4MB
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm4MB

three_bytes_emit_remainder_encodeBetterBlockAsm4MB:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm4MB

two_bytes_emit_remainder_encodeBetterBlockAsm4MB:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBetterBlockAsm4MB
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm4MB

one_byte_emit_remainder_encodeBetterBlockAsm4MB:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBetterBlockAsm4MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB

memmove_long_emit_remainder_encodeBetterBlockAsm4MB:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBetterBlockAsm12B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000280, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm12B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBetterBlockAsm12B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -6(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBetterBlockAsm12B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm12B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x34, R11
	MOVL  (AX)(R10*4), SI
	MOVL  65536(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 65536(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm12B
	CMPQ  R11, DI
	JNE   no_short_found_encodeBetterBlockAsm12B
	MOVL  R8, SI
	JMP   candidate_match_encodeBetterBlockAsm12B

no_short_found_encodeBetterBlockAsm12B:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm12B
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm12B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBetterBlockAsm12B

candidateS_match_encodeBetterBlockAsm12B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeBetterBlockAsm12B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeBetterBlockAsm12B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm12B

match_extend_back_loop_encodeBetterBlockAsm12B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBetterBlockAsm12B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBetterBlockAsm12B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm12B
	JMP  match_extend_back_loop_encodeBetterBlockAsm12B

match_extend_back_end_encodeBetterBlockAsm12B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm12B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeBetterBlockAsm12B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B

matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm12B

matchlen_match8_match_nolit_encodeBetterBlockAsm12B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm12B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm12B

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm12B

matchlen_match4_match_nolit_encodeBetterBlockAsm12B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm12B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm12B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeBetterBlockAsm12B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm12B
	JB   match_nolit_end_encodeBetterBlockAsm12B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm12B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeBetterBlockAsm12B

matchlen_match1_match_nolit_encodeBetterBlockAsm12B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeBetterBlockAsm12B
	LEAL 1(R12), R12

match_nolit_end_encodeBetterBlockAsm12B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL 16(SP), R8
	JEQ  match_is_repeat_encodeBetterBlockAsm12B
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeBetterBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeBetterBlockAsm12B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm12B
	JB   three_bytes_match_emit_encodeBetterBlockAsm12B

three_bytes_match_emit_encodeBetterBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm12B

two_bytes_match_emit_encodeBetterBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeBetterBlockAsm12B
	JMP  memmove_long_match_emit_encodeBetterBlockAsm12B

one_byte_match_emit_encodeBetterBlockAsm12B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm12B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeBetterBlockAsm12B

memmove_long_match_emit_encodeBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeBetterBlockAsm12B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B
	CMPL R8, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBetterBlockAsm12B
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R12

	// emitRepeat
	LEAL -4(R12), R12
	JMP  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

long_offset_short_match_nolit_encodeBetterBlockAsm12B:
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm12B
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm12B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

emit_copy_three_match_nolit_encodeBetterBlockAsm12B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

match_is_repeat_encodeBetterBlockAsm12B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm12B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm12B
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm12B

three_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm12B

two_bytes_match_emit_repeat_encodeBetterBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_repeat_encodeBetterBlockAsm12B
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm12B

one_byte_match_emit_repeat_encodeBetterBlockAsm12B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_repeat_encodeBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B

memmove_long_match_emit_repeat_encodeBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B

cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm12B

repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm12B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm12B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm12B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x32, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x34, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x32, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x34, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 65536(AX)(R11*4)
	MOVL  R14, 65536(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeBetterBlockAsm12B:
	CMPQ  R8, R9
	JAE   search_loop_encodeBetterBlockAsm12B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeBetterBlockAsm12B

emit_remainder_encodeBetterBlockAsm12B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm12B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm12B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBetterBlockAsm12B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm12B
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm12B

three_bytes_emit_remainder_encodeBetterBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm12B

two_bytes_emit_remainder_encodeBetterBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBetterBlockAsm12B
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm12B

one_byte_emit_remainder_encodeBetterBlockAsm12B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBetterBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm12B

memmove_long_emit_remainder_encodeBetterBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm12B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBetterBlockAsm10B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x000000a0, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm10B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBetterBlockAsm10B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -6(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBetterBlockAsm10B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm10B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x36, R11
	MOVL  (AX)(R10*4), SI
	MOVL  16384(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 16384(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm10B
	CMPQ  R11, DI
	JNE   no_short_found_encodeBetterBlockAsm10B
	MOVL  R8, SI
	JMP   candidate_match_encodeBetterBlockAsm10B

no_short_found_encodeBetterBlockAsm10B:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm10B
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm10B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBetterBlockAsm10B

candidateS_match_encodeBetterBlockAsm10B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeBetterBlockAsm10B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeBetterBlockAsm10B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm10B

match_extend_back_loop_encodeBetterBlockAsm10B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBetterBlockAsm10B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBetterBlockAsm10B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm10B
	JMP  match_extend_back_loop_encodeBetterBlockAsm10B

match_extend_back_end_encodeBetterBlockAsm10B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm10B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeBetterBlockAsm10B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B

matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm10B

matchlen_match8_match_nolit_encodeBetterBlockAsm10B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm10B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm10B

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm10B

matchlen_match4_match_nolit_encodeBetterBlockAsm10B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm10B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm10B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeBetterBlockAsm10B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm10B
	JB   match_nolit_end_encodeBetterBlockAsm10B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm10B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeBetterBlockAsm10B

matchlen_match1_match_nolit_encodeBetterBlockAsm10B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeBetterBlockAsm10B
	LEAL 1(R12), R12

match_nolit_end_encodeBetterBlockAsm10B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL 16(SP), R8
	JEQ  match_is_repeat_encodeBetterBlockAsm10B
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeBetterBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeBetterBlockAsm10B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm10B
	JB   three_bytes_match_emit_encodeBetterBlockAsm10B

three_bytes_match_emit_encodeBetterBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm10B

two_bytes_match_emit_encodeBetterBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeBetterBlockAsm10B
	JMP  memmove_long_match_emit_encodeBetterBlockAsm10B

one_byte_match_emit_encodeBetterBlockAsm10B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm10B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeBetterBlockAsm10B

memmove_long_match_emit_encodeBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeBetterBlockAsm10B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B
	CMPL R8, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBetterBlockAsm10B
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R12

	// emitRepeat
	LEAL -4(R12), R12
	JMP  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

long_offset_short_match_nolit_encodeBetterBlockAsm10B:
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm10B
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm10B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

emit_copy_three_match_nolit_encodeBetterBlockAsm10B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

match_is_repeat_encodeBetterBlockAsm10B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm10B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm10B
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm10B

three_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm10B

two_bytes_match_emit_repeat_encodeBetterBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_repeat_encodeBetterBlockAsm10B
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm10B

one_byte_match_emit_repeat_encodeBetterBlockAsm10B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_repeat_encodeBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B

memmove_long_match_emit_repeat_encodeBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B
	CMPL R8, $0x00000800
	JB   repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B

cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm10B

repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B:
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm10B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm10B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm10B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x34, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x36, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x34, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x36, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 16384(AX)(R11*4)
	MOVL  R14, 16384(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeBetterBlockAsm10B:
	CMPQ  R8, R9
	JAE   search_loop_encodeBetterBlockAsm10B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x34, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x34, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeBetterBlockAsm10B

emit_remainder_encodeBetterBlockAsm10B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm10B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm10B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBetterBlockAsm10B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm10B
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm10B

three_bytes_emit_remainder_encodeBetterBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm10B

two_bytes_emit_remainder_encodeBetterBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBetterBlockAsm10B
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm10B

one_byte_emit_remainder_encodeBetterBlockAsm10B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBetterBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm10B

memmove_long_emit_remainder_encodeBetterBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm10B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
// Requires: BMI, SSE2
TEXT ·encodeBetterBlockAsm8B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000028, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeBetterBlockAsm8B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeBetterBlockAsm8B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -6(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeBetterBlockAsm8B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x04, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeBetterBlockAsm8B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x38, R11
	MOVL  (AX)(R10*4), SI
	MOVL  4096(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 4096(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeBetterBlockAsm8B
	CMPQ  R11, DI
	JNE   no_short_found_encodeBetterBlockAsm8B
	MOVL  R8, SI
	JMP   candidate_match_encodeBetterBlockAsm8B

no_short_found_encodeBetterBlockAsm8B:
	CMPL R10, DI
	JEQ  candidate_match_encodeBetterBlockAsm8B
	CMPL R11, DI
	JEQ  candidateS_match_encodeBetterBlockAsm8B
	MOVL 20(SP), DX
	JMP  search_loop_encodeBetterBlockAsm8B

candidateS_match_encodeBetterBlockAsm8B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeBetterBlockAsm8B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeBetterBlockAsm8B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeBetterBlockAsm8B

match_extend_back_loop_encodeBetterBlockAsm8B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeBetterBlockAsm8B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeBetterBlockAsm8B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeBetterBlockAsm8B
	JMP  match_extend_back_loop_encodeBetterBlockAsm8B

match_extend_back_end_encodeBetterBlockAsm8B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeBetterBlockAsm8B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeBetterBlockAsm8B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B

matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm8B

matchlen_match8_match_nolit_encodeBetterBlockAsm8B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeBetterBlockAsm8B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeBetterBlockAsm8B

matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeBetterBlockAsm8B

matchlen_match4_match_nolit_encodeBetterBlockAsm8B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeBetterBlockAsm8B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeBetterBlockAsm8B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeBetterBlockAsm8B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeBetterBlockAsm8B
	JB   match_nolit_end_encodeBetterBlockAsm8B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeBetterBlockAsm8B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeBetterBlockAsm8B

matchlen_match1_match_nolit_encodeBetterBlockAsm8B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeBetterBlockAsm8B
	LEAL 1(R12), R12

match_nolit_end_encodeBetterBlockAsm8B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL 16(SP), R8
	JEQ  match_is_repeat_encodeBetterBlockAsm8B
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeBetterBlockAsm8B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeBetterBlockAsm8B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeBetterBlockAsm8B
	JB   three_bytes_match_emit_encodeBetterBlockAsm8B

three_bytes_match_emit_encodeBetterBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeBetterBlockAsm8B

two_bytes_match_emit_encodeBetterBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeBetterBlockAsm8B
	JMP  memmove_long_match_emit_encodeBetterBlockAsm8B

one_byte_match_emit_encodeBetterBlockAsm8B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeBetterBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x04
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4
	CMPQ R9, $0x08
	JB   emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4:
	MOVL (R10), R11
	MOVL R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7:
	MOVL (R10), R11
	MOVL -4(R10)(R9*1), R10
	MOVL R11, (CX)
	MOVL R10, -4(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeBetterBlockAsm8B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeBetterBlockAsm8B

memmove_long_match_emit_encodeBetterBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeBetterBlockAsm8B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B
	CMPL R8, $0x00000800
	JAE  long_offset_short_match_nolit_encodeBetterBlockAsm8B
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	SUBL $0x08, R12

	// emitRepeat
	LEAL -4(R12), R12
	JMP  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

long_offset_short_match_nolit_encodeBetterBlockAsm8B:
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short

cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeBetterBlockAsm8B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

emit_copy_three_match_nolit_encodeBetterBlockAsm8B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

match_is_repeat_encodeBetterBlockAsm8B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_repeat_encodeBetterBlockAsm8B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_repeat_encodeBetterBlockAsm8B
	JB   three_bytes_match_emit_repeat_encodeBetterBlockAsm8B

three_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm8B

two_bytes_match_emit_repeat_encodeBetterBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_repeat_encodeBetterBlockAsm8B
	JMP  memmove_long_match_emit_repeat_encodeBetterBlockAsm8B

one_byte_match_emit_repeat_encodeBetterBlockAsm8B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_repeat_encodeBetterBlockAsm8B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x04
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4
	CMPQ R8, $0x08
	JB   emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4:
	MOVL (R9), R10
	MOVL R10, (CX)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7:
	MOVL (R9), R10
	MOVL -4(R9)(R8*1), R9
	MOVL R10, (CX)
	MOVL R9, -4(CX)(R8*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B

emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B

memmove_long_match_emit_repeat_encodeBetterBlockAsm8B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R13
	SUBQ  R10, R13
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(R9)(R13*1), R10
	LEAQ  -32(CX)(R13*1), R14

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R10
	ADDQ  $0x20, R13
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(R9)(R13*1), X4
	MOVOU -16(R9)(R13*1), X5
	MOVOA X4, -32(CX)(R13*1)
	MOVOA X5, -16(CX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitRepeat
	MOVL R12, SI
	LEAL -4(R12), R12
	CMPL SI, $0x08
	JBE  repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B

cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B:
	CMPL R12, $0x00000104
	JB   repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B
	LEAL -256(R12), R12
	MOVW $0x0019, (CX)
	MOVW R12, 2(CX)
	ADDQ $0x04, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B:
	LEAL -4(R12), R12
	MOVW $0x0015, (CX)
	MOVB R12, 2(CX)
	ADDQ $0x03, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B

repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B:
	SHLL $0x02, R12
	ORL  $0x01, R12
	MOVW R12, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeBetterBlockAsm8B
	XORQ SI, SI
	LEAL 1(SI)(R12*4), R12
	MOVB R8, 1(CX)
	SARL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, R12
	MOVB R12, (CX)
	ADDQ $0x02, CX

match_nolit_emitcopy_end_encodeBetterBlockAsm8B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeBetterBlockAsm8B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeBetterBlockAsm8B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x38, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x36, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x38, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 4096(AX)(R11*4)
	MOVL  R14, 4096(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeBetterBlockAsm8B:
	CMPQ  R8, R9
	JAE   search_loop_encodeBetterBlockAsm8B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x36, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x36, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeBetterBlockAsm8B

emit_remainder_encodeBetterBlockAsm8B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeBetterBlockAsm8B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeBetterBlockAsm8B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeBetterBlockAsm8B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeBetterBlockAsm8B
	JB   three_bytes_emit_remainder_encodeBetterBlockAsm8B

three_bytes_emit_remainder_encodeBetterBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm8B

two_bytes_emit_remainder_encodeBetterBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeBetterBlockAsm8B
	JMP  memmove_long_emit_remainder_encodeBetterBlockAsm8B

one_byte_emit_remainder_encodeBetterBlockAsm8B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeBetterBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeBetterBlockAsm8B

memmove_long_emit_remainder_encodeBetterBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeBetterBlockAsm8B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBlockAsm:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBlockAsm
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ R9, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeSnappyBlockAsm
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeSnappyBlockAsm

repeat_extend_back_loop_encodeSnappyBlockAsm:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeSnappyBlockAsm
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeSnappyBlockAsm
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeSnappyBlockAsm

repeat_extend_back_end_encodeSnappyBlockAsm:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 5(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeSnappyBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeSnappyBlockAsm:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeSnappyBlockAsm
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeSnappyBlockAsm
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeSnappyBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_encodeSnappyBlockAsm
	CMPL SI, $0x01000000
	JB   four_bytes_repeat_emit_encodeSnappyBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm

four_bytes_repeat_emit_encodeSnappyBlockAsm:
	MOVL SI, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm

three_bytes_repeat_emit_encodeSnappyBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm

two_bytes_repeat_emit_encodeSnappyBlockAsm:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeSnappyBlockAsm
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm

one_byte_repeat_emit_encodeSnappyBlockAsm:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeSnappyBlockAsm:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8:
	MOVQ (R9), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_repeat_emit_encodeSnappyBlockAsm

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_repeat_emit_encodeSnappyBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeSnappyBlockAsm

memmove_long_repeat_emit_encodeSnappyBlockAsm:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(R9)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R9)(R12*1), X4
	MOVOU -16(R9)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R8, R12
	JAE   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeSnappyBlockAsm:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_encodeSnappyBlockAsm
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm

matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm

matchlen_match8_repeat_extend_encodeSnappyBlockAsm:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeSnappyBlockAsm
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeSnappyBlockAsm

matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm

matchlen_match4_repeat_extend_encodeSnappyBlockAsm:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeSnappyBlockAsm
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeSnappyBlockAsm
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeSnappyBlockAsm:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeSnappyBlockAsm
	JB   repeat_extend_forward_end_encodeSnappyBlockAsm
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeSnappyBlockAsm
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeSnappyBlockAsm

matchlen_match1_repeat_extend_encodeSnappyBlockAsm:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeSnappyBlockAsm
	LEAL 1(R11), R11

repeat_extend_forward_end_encodeSnappyBlockAsm:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
	CMPL DI, $0x00010000
	JB   two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm

four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm:
	CMPL SI, $0x40
	JBE  four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
	MOVB $0xff, (CX)
	MOVL DI, 1(CX)
	LEAL -64(SI), SI
	ADDQ $0x05, CX
	CMPL SI, $0x04
	JB   four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm
	JMP  four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm

four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm:
	TESTL SI, SI
	JZ    repeat_end_emit_encodeSnappyBlockAsm
	XORL  R8, R8
	LEAL  -1(R8)(SI*4), SI
	MOVB  SI, (CX)
	MOVL  DI, 1(CX)
	ADDQ  $0x05, CX
	JMP   repeat_end_emit_encodeSnappyBlockAsm

two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm

two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeSnappyBlockAsm

emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeSnappyBlockAsm:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeSnappyBlockAsm

no_repeat_found_encodeSnappyBlockAsm:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeSnappyBlockAsm
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeSnappyBlockAsm
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeSnappyBlockAsm
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBlockAsm

candidate3_match_encodeSnappyBlockAsm:
	ADDL $0x02, DX
	JMP  candidate_match_encodeSnappyBlockAsm

candidate2_match_encodeSnappyBlockAsm:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeSnappyBlockAsm:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBlockAsm

match_extend_back_loop_encodeSnappyBlockAsm:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBlockAsm
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBlockAsm
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBlockAsm
	JMP  match_extend_back_loop_encodeSnappyBlockAsm

match_extend_back_end_encodeSnappyBlockAsm:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 5(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBlockAsm:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeSnappyBlockAsm
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBlockAsm
	CMPL R8, $0x00010000
	JB   three_bytes_match_emit_encodeSnappyBlockAsm
	CMPL R8, $0x01000000
	JB   four_bytes_match_emit_encodeSnappyBlockAsm
	MOVB $0xfc, (CX)
	MOVL R8, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm

four_bytes_match_emit_encodeSnappyBlockAsm:
	MOVL R8, R10
	SHRL $0x10, R10
	MOVB $0xf8, (CX)
	MOVW R8, 1(CX)
	MOVB R10, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm

three_bytes_match_emit_encodeSnappyBlockAsm:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm

two_bytes_match_emit_encodeSnappyBlockAsm:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeSnappyBlockAsm
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm

one_byte_match_emit_encodeSnappyBlockAsm:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBlockAsm:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBlockAsm:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBlockAsm

memmove_long_match_emit_encodeSnappyBlockAsm:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeSnappyBlockAsm:
match_nolit_loop_encodeSnappyBlockAsm:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBlockAsm
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBlockAsm
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm

matchlen_bsf_16match_nolit_encodeSnappyBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm

matchlen_match8_match_nolit_encodeSnappyBlockAsm:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBlockAsm
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeSnappyBlockAsm

matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm

matchlen_match4_match_nolit_encodeSnappyBlockAsm:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBlockAsm
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeSnappyBlockAsm
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeSnappyBlockAsm:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBlockAsm
	JB   match_nolit_end_encodeSnappyBlockAsm
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeSnappyBlockAsm
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeSnappyBlockAsm

matchlen_match1_match_nolit_encodeSnappyBlockAsm:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeSnappyBlockAsm
	LEAL 1(R10), R10

match_nolit_end_encodeSnappyBlockAsm:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL SI, $0x00010000
	JB   two_byte_offset_match_nolit_encodeSnappyBlockAsm

four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm:
	CMPL R10, $0x40
	JBE  four_bytes_remain_match_nolit_encodeSnappyBlockAsm
	MOVB $0xff, (CX)
	MOVL SI, 1(CX)
	LEAL -64(R10), R10
	ADDQ $0x05, CX
	CMPL R10, $0x04
	JB   four_bytes_remain_match_nolit_encodeSnappyBlockAsm
	JMP  four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm

four_bytes_remain_match_nolit_encodeSnappyBlockAsm:
	TESTL R10, R10
	JZ    match_nolit_emitcopy_end_encodeSnappyBlockAsm
	XORL  DI, DI
	LEAL  -1(DI)(R10*4), R10
	MOVB  R10, (CX)
	MOVL  SI, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeSnappyBlockAsm

two_byte_offset_match_nolit_encodeSnappyBlockAsm:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBlockAsm
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBlockAsm

two_byte_offset_short_match_nolit_encodeSnappyBlockAsm:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBlockAsm

emit_copy_three_match_nolit_encodeSnappyBlockAsm:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBlockAsm
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBlockAsm:
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, SI
	IMULQ R9, SI
	SHRQ  $0x32, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeSnappyBlockAsm
	INCL  DX
	JMP   search_loop_encodeSnappyBlockAsm

emit_remainder_encodeSnappyBlockAsm:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 5(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBlockAsm
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBlockAsm
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBlockAsm
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeSnappyBlockAsm
	CMPL DX, $0x01000000
	JB   four_bytes_emit_remainder_encodeSnappyBlockAsm
	MOVB $0xfc, (CX)
	MOVL DX, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm

four_bytes_emit_remainder_encodeSnappyBlockAsm:
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm

three_bytes_emit_remainder_encodeSnappyBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm

two_bytes_emit_remainder_encodeSnappyBlockAsm:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBlockAsm
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm

one_byte_emit_remainder_encodeSnappyBlockAsm:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBlockAsm

memmove_long_emit_remainder_encodeSnappyBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBlockAsm64K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBlockAsm64K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBlockAsm64K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBlockAsm64K:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBlockAsm64K
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ R9, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeSnappyBlockAsm64K
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeSnappyBlockAsm64K

repeat_extend_back_loop_encodeSnappyBlockAsm64K:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeSnappyBlockAsm64K
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeSnappyBlockAsm64K
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeSnappyBlockAsm64K

repeat_extend_back_end_encodeSnappyBlockAsm64K:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeSnappyBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeSnappyBlockAsm64K:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeSnappyBlockAsm64K
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeSnappyBlockAsm64K
	JB   three_bytes_repeat_emit_encodeSnappyBlockAsm64K

three_bytes_repeat_emit_encodeSnappyBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm64K

two_bytes_repeat_emit_encodeSnappyBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeSnappyBlockAsm64K
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm64K

one_byte_repeat_emit_encodeSnappyBlockAsm64K:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeSnappyBlockAsm64K:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8:
	MOVQ (R9), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K

memmove_long_repeat_emit_encodeSnappyBlockAsm64K:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	LEAQ  -32(R9)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R9)(R12*1), X4
	MOVOU -16(R9)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R8, R12
	JAE   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K

matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm64K

matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K

matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm64K

matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
	JB   repeat_extend_forward_end_encodeSnappyBlockAsm64K
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeSnappyBlockAsm64K

matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeSnappyBlockAsm64K
	LEAL 1(R11), R11

repeat_extend_forward_end_encodeSnappyBlockAsm64K:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K

two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeSnappyBlockAsm64K

emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeSnappyBlockAsm64K:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeSnappyBlockAsm64K

no_repeat_found_encodeSnappyBlockAsm64K:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeSnappyBlockAsm64K
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeSnappyBlockAsm64K
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeSnappyBlockAsm64K
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBlockAsm64K

candidate3_match_encodeSnappyBlockAsm64K:
	ADDL $0x02, DX
	JMP  candidate_match_encodeSnappyBlockAsm64K

candidate2_match_encodeSnappyBlockAsm64K:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeSnappyBlockAsm64K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBlockAsm64K

match_extend_back_loop_encodeSnappyBlockAsm64K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBlockAsm64K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBlockAsm64K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBlockAsm64K
	JMP  match_extend_back_loop_encodeSnappyBlockAsm64K

match_extend_back_end_encodeSnappyBlockAsm64K:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBlockAsm64K:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBlockAsm64K
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeSnappyBlockAsm64K
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBlockAsm64K
	JB   three_bytes_match_emit_encodeSnappyBlockAsm64K

three_bytes_match_emit_encodeSnappyBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm64K

two_bytes_match_emit_encodeSnappyBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeSnappyBlockAsm64K
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm64K

one_byte_match_emit_encodeSnappyBlockAsm64K:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBlockAsm64K:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBlockAsm64K:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBlockAsm64K

memmove_long_match_emit_encodeSnappyBlockAsm64K:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeSnappyBlockAsm64K:
match_nolit_loop_encodeSnappyBlockAsm64K:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBlockAsm64K
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K

matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm64K

matchlen_match8_match_nolit_encodeSnappyBlockAsm64K:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBlockAsm64K
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeSnappyBlockAsm64K

matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm64K

matchlen_match4_match_nolit_encodeSnappyBlockAsm64K:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeSnappyBlockAsm64K
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeSnappyBlockAsm64K:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
	JB   match_nolit_end_encodeSnappyBlockAsm64K
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeSnappyBlockAsm64K
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeSnappyBlockAsm64K

matchlen_match1_match_nolit_encodeSnappyBlockAsm64K:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeSnappyBlockAsm64K
	LEAL 1(R10), R10

match_nolit_end_encodeSnappyBlockAsm64K:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBlockAsm64K:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBlockAsm64K

two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm64K
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBlockAsm64K

emit_copy_three_match_nolit_encodeSnappyBlockAsm64K:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBlockAsm64K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBlockAsm64K
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBlockAsm64K:
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x32, R8
	SHLQ  $0x10, SI
	IMULQ R9, SI
	SHRQ  $0x32, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeSnappyBlockAsm64K
	INCL  DX
	JMP   search_loop_encodeSnappyBlockAsm64K

emit_remainder_encodeSnappyBlockAsm64K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBlockAsm64K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBlockAsm64K
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBlockAsm64K
	JB   three_bytes_emit_remainder_encodeSnappyBlockAsm64K

three_bytes_emit_remainder_encodeSnappyBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm64K

two_bytes_emit_remainder_encodeSnappyBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBlockAsm64K
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm64K

one_byte_emit_remainder_encodeSnappyBlockAsm64K:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K

memmove_long_emit_remainder_encodeSnappyBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBlockAsm12B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000080, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBlockAsm12B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBlockAsm12B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBlockAsm12B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBlockAsm12B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x000000cf1bbcdcbb, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x18, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	SHLQ  $0x18, R11
	IMULQ R9, R11
	SHRQ  $0x34, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x18, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeSnappyBlockAsm12B
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeSnappyBlockAsm12B

repeat_extend_back_loop_encodeSnappyBlockAsm12B:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeSnappyBlockAsm12B
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeSnappyBlockAsm12B
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeSnappyBlockAsm12B

repeat_extend_back_end_encodeSnappyBlockAsm12B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeSnappyBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeSnappyBlockAsm12B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeSnappyBlockAsm12B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeSnappyBlockAsm12B
	JB   three_bytes_repeat_emit_encodeSnappyBlockAsm12B

three_bytes_repeat_emit_encodeSnappyBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm12B

two_bytes_repeat_emit_encodeSnappyBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeSnappyBlockAsm12B
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm12B

one_byte_repeat_emit_encodeSnappyBlockAsm12B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeSnappyBlockAsm12B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8:
	MOVQ (R9), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B

memmove_long_repeat_emit_encodeSnappyBlockAsm12B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(R9)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(R9)(R12*1), X4
	MOVOU -16(R9)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R8, R12
	JAE   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B

matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm12B

matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B

matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm12B

matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
	JB   repeat_extend_forward_end_encodeSnappyBlockAsm12B
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeSnappyBlockAsm12B

matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeSnappyBlockAsm12B
	LEAL 1(R11), R11

repeat_extend_forward_end_encodeSnappyBlockAsm12B:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B

two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeSnappyBlockAsm12B

emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeSnappyBlockAsm12B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeSnappyBlockAsm12B

no_repeat_found_encodeSnappyBlockAsm12B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeSnappyBlockAsm12B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeSnappyBlockAsm12B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeSnappyBlockAsm12B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBlockAsm12B

candidate3_match_encodeSnappyBlockAsm12B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeSnappyBlockAsm12B

candidate2_match_encodeSnappyBlockAsm12B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeSnappyBlockAsm12B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBlockAsm12B

match_extend_back_loop_encodeSnappyBlockAsm12B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBlockAsm12B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBlockAsm12B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBlockAsm12B
	JMP  match_extend_back_loop_encodeSnappyBlockAsm12B

match_extend_back_end_encodeSnappyBlockAsm12B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBlockAsm12B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeSnappyBlockAsm12B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBlockAsm12B
	JB   three_bytes_match_emit_encodeSnappyBlockAsm12B

three_bytes_match_emit_encodeSnappyBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm12B

two_bytes_match_emit_encodeSnappyBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeSnappyBlockAsm12B
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm12B

one_byte_match_emit_encodeSnappyBlockAsm12B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBlockAsm12B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBlockAsm12B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBlockAsm12B

memmove_long_match_emit_encodeSnappyBlockAsm12B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeSnappyBlockAsm12B:
match_nolit_loop_encodeSnappyBlockAsm12B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBlockAsm12B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B

matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm12B

matchlen_match8_match_nolit_encodeSnappyBlockAsm12B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBlockAsm12B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeSnappyBlockAsm12B

matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm12B

matchlen_match4_match_nolit_encodeSnappyBlockAsm12B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeSnappyBlockAsm12B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeSnappyBlockAsm12B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
	JB   match_nolit_end_encodeSnappyBlockAsm12B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeSnappyBlockAsm12B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeSnappyBlockAsm12B

matchlen_match1_match_nolit_encodeSnappyBlockAsm12B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeSnappyBlockAsm12B
	LEAL 1(R10), R10

match_nolit_end_encodeSnappyBlockAsm12B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBlockAsm12B:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBlockAsm12B

two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm12B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBlockAsm12B

emit_copy_three_match_nolit_encodeSnappyBlockAsm12B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBlockAsm12B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBlockAsm12B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBlockAsm12B:
	MOVQ  $0x000000cf1bbcdcbb, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x18, R8
	IMULQ R9, R8
	SHRQ  $0x34, R8
	SHLQ  $0x18, SI
	IMULQ R9, SI
	SHRQ  $0x34, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeSnappyBlockAsm12B
	INCL  DX
	JMP   search_loop_encodeSnappyBlockAsm12B

emit_remainder_encodeSnappyBlockAsm12B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBlockAsm12B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBlockAsm12B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBlockAsm12B
	JB   three_bytes_emit_remainder_encodeSnappyBlockAsm12B

three_bytes_emit_remainder_encodeSnappyBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm12B

two_bytes_emit_remainder_encodeSnappyBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBlockAsm12B
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm12B

one_byte_emit_remainder_encodeSnappyBlockAsm12B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B

memmove_long_emit_remainder_encodeSnappyBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBlockAsm10B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000020, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBlockAsm10B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBlockAsm10B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBlockAsm10B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBlockAsm10B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ R9, R11
	SHRQ  $0x36, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeSnappyBlockAsm10B
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeSnappyBlockAsm10B

repeat_extend_back_loop_encodeSnappyBlockAsm10B:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeSnappyBlockAsm10B
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeSnappyBlockAsm10B
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeSnappyBlockAsm10B

repeat_extend_back_end_encodeSnappyBlockAsm10B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeSnappyBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeSnappyBlockAsm10B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeSnappyBlockAsm10B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeSnappyBlockAsm10B
	JB   three_bytes_repeat_emit_encodeSnappyBlockAsm10B

three_bytes_repeat_emit_encodeSnappyBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm10B

two_bytes_repeat_emit_encodeSnappyBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeSnappyBlockAsm10B
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm10B

one_byte_repeat_emit_encodeSnappyBlockAsm10B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeSnappyBlockAsm10B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8:
	MOVQ (R9), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B

memmove_long_repeat_emit_encodeSnappyBlockAsm10B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(R9)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(R9)(R12*1), X4
	MOVOU -16(R9)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R8, R12
	JAE   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B

matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm10B

matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B

matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm10B

matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
	JB   repeat_extend_forward_end_encodeSnappyBlockAsm10B
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeSnappyBlockAsm10B

matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeSnappyBlockAsm10B
	LEAL 1(R11), R11

repeat_extend_forward_end_encodeSnappyBlockAsm10B:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B

two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeSnappyBlockAsm10B

emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeSnappyBlockAsm10B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeSnappyBlockAsm10B

no_repeat_found_encodeSnappyBlockAsm10B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeSnappyBlockAsm10B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeSnappyBlockAsm10B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeSnappyBlockAsm10B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBlockAsm10B

candidate3_match_encodeSnappyBlockAsm10B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeSnappyBlockAsm10B

candidate2_match_encodeSnappyBlockAsm10B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeSnappyBlockAsm10B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBlockAsm10B

match_extend_back_loop_encodeSnappyBlockAsm10B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBlockAsm10B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBlockAsm10B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBlockAsm10B
	JMP  match_extend_back_loop_encodeSnappyBlockAsm10B

match_extend_back_end_encodeSnappyBlockAsm10B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBlockAsm10B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeSnappyBlockAsm10B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBlockAsm10B
	JB   three_bytes_match_emit_encodeSnappyBlockAsm10B

three_bytes_match_emit_encodeSnappyBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm10B

two_bytes_match_emit_encodeSnappyBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeSnappyBlockAsm10B
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm10B

one_byte_match_emit_encodeSnappyBlockAsm10B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBlockAsm10B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBlockAsm10B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBlockAsm10B

memmove_long_match_emit_encodeSnappyBlockAsm10B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeSnappyBlockAsm10B:
match_nolit_loop_encodeSnappyBlockAsm10B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBlockAsm10B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B

matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm10B

matchlen_match8_match_nolit_encodeSnappyBlockAsm10B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBlockAsm10B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeSnappyBlockAsm10B

matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm10B

matchlen_match4_match_nolit_encodeSnappyBlockAsm10B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeSnappyBlockAsm10B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeSnappyBlockAsm10B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
	JB   match_nolit_end_encodeSnappyBlockAsm10B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeSnappyBlockAsm10B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeSnappyBlockAsm10B

matchlen_match1_match_nolit_encodeSnappyBlockAsm10B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeSnappyBlockAsm10B
	LEAL 1(R10), R10

match_nolit_end_encodeSnappyBlockAsm10B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBlockAsm10B:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBlockAsm10B

two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm10B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBlockAsm10B

emit_copy_three_match_nolit_encodeSnappyBlockAsm10B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBlockAsm10B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBlockAsm10B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBlockAsm10B:
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x20, R8
	IMULQ R9, R8
	SHRQ  $0x36, R8
	SHLQ  $0x20, SI
	IMULQ R9, SI
	SHRQ  $0x36, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeSnappyBlockAsm10B
	INCL  DX
	JMP   search_loop_encodeSnappyBlockAsm10B

emit_remainder_encodeSnappyBlockAsm10B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBlockAsm10B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBlockAsm10B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBlockAsm10B
	JB   three_bytes_emit_remainder_encodeSnappyBlockAsm10B

three_bytes_emit_remainder_encodeSnappyBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm10B

two_bytes_emit_remainder_encodeSnappyBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBlockAsm10B
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm10B

one_byte_emit_remainder_encodeSnappyBlockAsm10B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B

memmove_long_emit_remainder_encodeSnappyBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBlockAsm8B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000008, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBlockAsm8B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBlockAsm8B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBlockAsm8B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x04, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBlockAsm8B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x38, R10
	SHLQ  $0x20, R11
	IMULQ R9, R11
	SHRQ  $0x38, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x38, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_encodeSnappyBlockAsm8B
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_encodeSnappyBlockAsm8B

repeat_extend_back_loop_encodeSnappyBlockAsm8B:
	CMPL DI, SI
	JBE  repeat_extend_back_end_encodeSnappyBlockAsm8B
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_encodeSnappyBlockAsm8B
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_encodeSnappyBlockAsm8B

repeat_extend_back_end_encodeSnappyBlockAsm8B:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_encodeSnappyBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

repeat_dst_size_check_encodeSnappyBlockAsm8B:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_encodeSnappyBlockAsm8B
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_encodeSnappyBlockAsm8B
	JB   three_bytes_repeat_emit_encodeSnappyBlockAsm8B

three_bytes_repeat_emit_encodeSnappyBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm8B

two_bytes_repeat_emit_encodeSnappyBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_encodeSnappyBlockAsm8B
	JMP  memmove_long_repeat_emit_encodeSnappyBlockAsm8B

one_byte_repeat_emit_encodeSnappyBlockAsm8B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_repeat_emit_encodeSnappyBlockAsm8B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8:
	MOVQ (R9), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
	MOVQ (R9), R10
	MOVQ -8(R9)(R8*1), R9
	MOVQ R10, (CX)
	MOVQ R9, -8(CX)(R8*1)
	JMP  memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
	MOVOU (R9), X0
	MOVOU -16(R9)(R8*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R8*1)
	JMP   memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)

memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B:
	MOVQ SI, CX
	JMP  emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B

memmove_long_repeat_emit_encodeSnappyBlockAsm8B:
	LEAQ (CX)(R8*1), SI

	// genMemMoveLong
	MOVOU (R9), X0
	MOVOU 16(R9), X1
	MOVOU -32(R9)(R8*1), X2
	MOVOU -16(R9)(R8*1), X3
	MOVQ  R8, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(R9)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(R9)(R12*1), X4
	MOVOU -16(R9)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R8, R12
	JAE   emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R8*1)
	MOVOU X3, -16(CX)(R8*1)
	MOVQ  SI, CX

emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B

matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm8B

matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B

matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_encodeSnappyBlockAsm8B

matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
	JB   repeat_extend_forward_end_encodeSnappyBlockAsm8B
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_encodeSnappyBlockAsm8B

matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_encodeSnappyBlockAsm8B
	LEAL 1(R11), R11

repeat_extend_forward_end_encodeSnappyBlockAsm8B:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B
	MOVB $0xee, (CX)
	MOVW DI, 1(CX)
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B

two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B
	LEAL -15(R8), R8
	MOVB DI, 1(CX)
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, R8
	MOVB R8, (CX)
	ADDQ $0x02, CX
	JMP  repeat_end_emit_encodeSnappyBlockAsm8B

emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B:
	LEAL -2(R8), R8
	MOVB R8, (CX)
	MOVW DI, 1(CX)
	ADDQ $0x03, CX

repeat_end_emit_encodeSnappyBlockAsm8B:
	MOVL DX, 12(SP)
	JMP  search_loop_encodeSnappyBlockAsm8B

no_repeat_found_encodeSnappyBlockAsm8B:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_encodeSnappyBlockAsm8B
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_encodeSnappyBlockAsm8B
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_encodeSnappyBlockAsm8B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBlockAsm8B

candidate3_match_encodeSnappyBlockAsm8B:
	ADDL $0x02, DX
	JMP  candidate_match_encodeSnappyBlockAsm8B

candidate2_match_encodeSnappyBlockAsm8B:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_encodeSnappyBlockAsm8B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBlockAsm8B

match_extend_back_loop_encodeSnappyBlockAsm8B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBlockAsm8B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBlockAsm8B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBlockAsm8B
	JMP  match_extend_back_loop_encodeSnappyBlockAsm8B

match_extend_back_end_encodeSnappyBlockAsm8B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBlockAsm8B:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBlockAsm8B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), R8
	CMPL R8, $0x3c
	JB   one_byte_match_emit_encodeSnappyBlockAsm8B
	CMPL R8, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBlockAsm8B
	JB   three_bytes_match_emit_encodeSnappyBlockAsm8B

three_bytes_match_emit_encodeSnappyBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm8B

two_bytes_match_emit_encodeSnappyBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB R8, 1(CX)
	ADDQ $0x02, CX
	CMPL R8, $0x40
	JB   memmove_match_emit_encodeSnappyBlockAsm8B
	JMP  memmove_long_match_emit_encodeSnappyBlockAsm8B

one_byte_match_emit_encodeSnappyBlockAsm8B:
	SHLB $0x02, R8
	MOVB R8, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBlockAsm8B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8:
	MOVQ (DI), R10
	MOVQ R10, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16:
	MOVQ (DI), R10
	MOVQ -8(DI)(R9*1), DI
	MOVQ R10, (CX)
	MOVQ DI, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32:
	MOVOU (DI), X0
	MOVOU -16(DI)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64:
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBlockAsm8B:
	MOVQ R8, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBlockAsm8B

memmove_long_match_emit_encodeSnappyBlockAsm8B:
	LEAQ (CX)(R9*1), R8

	// genMemMoveLong
	MOVOU (DI), X0
	MOVOU 16(DI), X1
	MOVOU -32(DI)(R9*1), X2
	MOVOU -16(DI)(R9*1), X3
	MOVQ  R9, R11
	SHRQ  $0x05, R11
	MOVQ  CX, R10
	ANDL  $0x0000001f, R10
	MOVQ  $0x00000040, R12
	SUBQ  R10, R12
	DECQ  R11
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(DI)(R12*1), R10
	LEAQ  -32(CX)(R12*1), R13

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back:
	MOVOU (R10), X4
	MOVOU 16(R10), X5
	MOVOA X4, (R13)
	MOVOA X5, 16(R13)
	ADDQ  $0x20, R13
	ADDQ  $0x20, R10
	ADDQ  $0x20, R12
	DECQ  R11
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(DI)(R12*1), X4
	MOVOU -16(DI)(R12*1), X5
	MOVOA X4, -32(CX)(R12*1)
	MOVOA X5, -16(CX)(R12*1)
	ADDQ  $0x20, R12
	CMPQ  R9, R12
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  R8, CX

emit_literal_done_match_emit_encodeSnappyBlockAsm8B:
match_nolit_loop_encodeSnappyBlockAsm8B:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBlockAsm8B
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B

matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm8B

matchlen_match8_match_nolit_encodeSnappyBlockAsm8B:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBlockAsm8B
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_encodeSnappyBlockAsm8B

matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_encodeSnappyBlockAsm8B

matchlen_match4_match_nolit_encodeSnappyBlockAsm8B:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_encodeSnappyBlockAsm8B
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_encodeSnappyBlockAsm8B:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
	JB   match_nolit_end_encodeSnappyBlockAsm8B
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_encodeSnappyBlockAsm8B
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_encodeSnappyBlockAsm8B

matchlen_match1_match_nolit_encodeSnappyBlockAsm8B:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_encodeSnappyBlockAsm8B
	LEAL 1(R10), R10

match_nolit_end_encodeSnappyBlockAsm8B:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBlockAsm8B:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B
	MOVB $0xee, (CX)
	MOVW SI, 1(CX)
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBlockAsm8B

two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBlockAsm8B
	LEAL -15(DI), DI
	MOVB SI, 1(CX)
	SHRL $0x08, SI
	SHLL $0x05, SI
	ORL  SI, DI
	MOVB DI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBlockAsm8B

emit_copy_three_match_nolit_encodeSnappyBlockAsm8B:
	LEAL -2(DI), DI
	MOVB DI, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBlockAsm8B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBlockAsm8B
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBlockAsm8B:
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x20, R8
	IMULQ R9, R8
	SHRQ  $0x38, R8
	SHLQ  $0x20, SI
	IMULQ R9, SI
	SHRQ  $0x38, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_encodeSnappyBlockAsm8B
	INCL  DX
	JMP   search_loop_encodeSnappyBlockAsm8B

emit_remainder_encodeSnappyBlockAsm8B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBlockAsm8B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBlockAsm8B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBlockAsm8B
	JB   three_bytes_emit_remainder_encodeSnappyBlockAsm8B

three_bytes_emit_remainder_encodeSnappyBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm8B

two_bytes_emit_remainder_encodeSnappyBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBlockAsm8B
	JMP  memmove_long_emit_remainder_encodeSnappyBlockAsm8B

one_byte_emit_remainder_encodeSnappyBlockAsm8B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B

memmove_long_emit_remainder_encodeSnappyBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00001200, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBetterBlockAsm:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBetterBlockAsm
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBetterBlockAsm:
	MOVL DX, SI
	SUBL 12(SP), SI
	SHRL $0x07, SI
	CMPL SI, $0x63
	JBE  check_maxskip_ok_encodeSnappyBetterBlockAsm
	LEAL 100(DX), SI
	JMP  check_maxskip_cont_encodeSnappyBetterBlockAsm

check_maxskip_ok_encodeSnappyBetterBlockAsm:
	LEAL 1(DX)(SI*1), SI

check_maxskip_cont_encodeSnappyBetterBlockAsm:
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBetterBlockAsm
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  (AX)(R10*4), SI
	MOVL  524288(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 524288(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm
	CMPQ  R11, DI
	JNE   no_short_found_encodeSnappyBetterBlockAsm
	MOVL  R8, SI
	JMP   candidate_match_encodeSnappyBetterBlockAsm

no_short_found_encodeSnappyBetterBlockAsm:
	CMPL R10, DI
	JEQ  candidate_match_encodeSnappyBetterBlockAsm
	CMPL R11, DI
	JEQ  candidateS_match_encodeSnappyBetterBlockAsm
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBetterBlockAsm

candidateS_match_encodeSnappyBetterBlockAsm:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x2f, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeSnappyBetterBlockAsm:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBetterBlockAsm

match_extend_back_loop_encodeSnappyBetterBlockAsm:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBetterBlockAsm
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBetterBlockAsm
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBetterBlockAsm
	JMP  match_extend_back_loop_encodeSnappyBetterBlockAsm

match_extend_back_end_encodeSnappyBetterBlockAsm:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 5(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBetterBlockAsm:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm

matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm

matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm

matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm

matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
	JB   match_nolit_end_encodeSnappyBetterBlockAsm
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeSnappyBetterBlockAsm

matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeSnappyBetterBlockAsm
	LEAL 1(R12), R12

match_nolit_end_encodeSnappyBetterBlockAsm:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	CMPL R12, $0x01
	JA   match_length_ok_encodeSnappyBetterBlockAsm
	CMPL R8, $0x0000ffff
	JBE  match_length_ok_encodeSnappyBetterBlockAsm
	MOVL 20(SP), DX
	INCL DX
	JMP  search_loop_encodeSnappyBetterBlockAsm

match_length_ok_encodeSnappyBetterBlockAsm:
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeSnappyBetterBlockAsm
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBetterBlockAsm
	CMPL SI, $0x00010000
	JB   three_bytes_match_emit_encodeSnappyBetterBlockAsm
	CMPL SI, $0x01000000
	JB   four_bytes_match_emit_encodeSnappyBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL SI, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm

four_bytes_match_emit_encodeSnappyBetterBlockAsm:
	MOVL SI, R11
	SHRL $0x10, R11
	MOVB $0xf8, (CX)
	MOVW SI, 1(CX)
	MOVB R11, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm

three_bytes_match_emit_encodeSnappyBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm

two_bytes_match_emit_encodeSnappyBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeSnappyBetterBlockAsm
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm

one_byte_match_emit_encodeSnappyBetterBlockAsm:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm

memmove_long_match_emit_encodeSnappyBetterBlockAsm:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeSnappyBetterBlockAsm:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
	CMPL R8, $0x00010000
	JB   two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm

four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R12, $0x40
	JBE  four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
	MOVB $0xff, (CX)
	MOVL R8, 1(CX)
	LEAL -64(R12), R12
	ADDQ $0x05, CX
	CMPL R12, $0x04
	JB   four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm
	JMP  four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm

four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm:
	TESTL R12, R12
	JZ    match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm
	XORL  SI, SI
	LEAL  -1(SI)(R12*4), R12
	MOVB  R12, (CX)
	MOVL  R8, 1(CX)
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm

two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm

two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm

emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBetterBlockAsm
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBetterBlockAsm:
	MOVQ  $0x00cf1bbcdcbfa563, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x32, R11
	SHLQ  $0x08, R12
	IMULQ SI, R12
	SHRQ  $0x2f, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x32, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 524288(AX)(R11*4)
	MOVL  R14, 524288(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeSnappyBetterBlockAsm:
	CMPQ  R8, R9
	JAE   search_loop_encodeSnappyBetterBlockAsm
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x2f, R10
	SHLQ  $0x08, R11
	IMULQ SI, R11
	SHRQ  $0x2f, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeSnappyBetterBlockAsm

emit_remainder_encodeSnappyBetterBlockAsm:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 5(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBetterBlockAsm
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBetterBlockAsm:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBetterBlockAsm
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBetterBlockAsm
	CMPL DX, $0x00010000
	JB   three_bytes_emit_remainder_encodeSnappyBetterBlockAsm
	CMPL DX, $0x01000000
	JB   four_bytes_emit_remainder_encodeSnappyBetterBlockAsm
	MOVB $0xfc, (CX)
	MOVL DX, 1(CX)
	ADDQ $0x05, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm

four_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
	MOVL DX, BX
	SHRL $0x10, BX
	MOVB $0xf8, (CX)
	MOVW DX, 1(CX)
	MOVB BL, 3(CX)
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm

three_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm

two_bytes_emit_remainder_encodeSnappyBetterBlockAsm:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBetterBlockAsm
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm

one_byte_emit_remainder_encodeSnappyBetterBlockAsm:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm

memmove_long_emit_remainder_encodeSnappyBetterBlockAsm:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000900, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBetterBlockAsm64K:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBetterBlockAsm64K
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBetterBlockAsm64K:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x07, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBetterBlockAsm64K
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x00cf1bbcdcbfa563, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x30, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x33, R11
	MOVL  (AX)(R10*4), SI
	MOVL  262144(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 262144(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm64K
	CMPQ  R11, DI
	JNE   no_short_found_encodeSnappyBetterBlockAsm64K
	MOVL  R8, SI
	JMP   candidate_match_encodeSnappyBetterBlockAsm64K

no_short_found_encodeSnappyBetterBlockAsm64K:
	CMPL R10, DI
	JEQ  candidate_match_encodeSnappyBetterBlockAsm64K
	CMPL R11, DI
	JEQ  candidateS_match_encodeSnappyBetterBlockAsm64K
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBetterBlockAsm64K

candidateS_match_encodeSnappyBetterBlockAsm64K:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x08, R10
	IMULQ R9, R10
	SHRQ  $0x30, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm64K
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeSnappyBetterBlockAsm64K:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBetterBlockAsm64K

match_extend_back_loop_encodeSnappyBetterBlockAsm64K:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBetterBlockAsm64K
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBetterBlockAsm64K
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBetterBlockAsm64K
	JMP  match_extend_back_loop_encodeSnappyBetterBlockAsm64K

match_extend_back_end_encodeSnappyBetterBlockAsm64K:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBetterBlockAsm64K:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K

matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm64K

matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K

matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm64K

matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
	JB   match_nolit_end_encodeSnappyBetterBlockAsm64K
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeSnappyBetterBlockAsm64K

matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeSnappyBetterBlockAsm64K
	LEAL 1(R12), R12

match_nolit_end_encodeSnappyBetterBlockAsm64K:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeSnappyBetterBlockAsm64K
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBetterBlockAsm64K
	JB   three_bytes_match_emit_encodeSnappyBetterBlockAsm64K

three_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm64K

two_bytes_match_emit_encodeSnappyBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeSnappyBetterBlockAsm64K
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm64K

one_byte_match_emit_encodeSnappyBetterBlockAsm64K:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBetterBlockAsm64K:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K

memmove_long_match_emit_encodeSnappyBetterBlockAsm64K:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K

two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K

emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBetterBlockAsm64K
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K:
	MOVQ  $0x00cf1bbcdcbfa563, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x30, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x33, R11
	SHLQ  $0x08, R12
	IMULQ SI, R12
	SHRQ  $0x30, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x33, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 262144(AX)(R11*4)
	MOVL  R14, 262144(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeSnappyBetterBlockAsm64K:
	CMPQ  R8, R9
	JAE   search_loop_encodeSnappyBetterBlockAsm64K
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x08, R10
	IMULQ SI, R10
	SHRQ  $0x30, R10
	SHLQ  $0x08, R11
	IMULQ SI, R11
	SHRQ  $0x30, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeSnappyBetterBlockAsm64K

emit_remainder_encodeSnappyBetterBlockAsm64K:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBetterBlockAsm64K
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBetterBlockAsm64K:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K
	JB   three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K

three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K

two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBetterBlockAsm64K
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K

one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBetterBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K

memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000280, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBetterBlockAsm12B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBetterBlockAsm12B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBetterBlockAsm12B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x06, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBetterBlockAsm12B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x34, R11
	MOVL  (AX)(R10*4), SI
	MOVL  65536(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 65536(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm12B
	CMPQ  R11, DI
	JNE   no_short_found_encodeSnappyBetterBlockAsm12B
	MOVL  R8, SI
	JMP   candidate_match_encodeSnappyBetterBlockAsm12B

no_short_found_encodeSnappyBetterBlockAsm12B:
	CMPL R10, DI
	JEQ  candidate_match_encodeSnappyBetterBlockAsm12B
	CMPL R11, DI
	JEQ  candidateS_match_encodeSnappyBetterBlockAsm12B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBetterBlockAsm12B

candidateS_match_encodeSnappyBetterBlockAsm12B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x32, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm12B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeSnappyBetterBlockAsm12B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBetterBlockAsm12B

match_extend_back_loop_encodeSnappyBetterBlockAsm12B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBetterBlockAsm12B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBetterBlockAsm12B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBetterBlockAsm12B
	JMP  match_extend_back_loop_encodeSnappyBetterBlockAsm12B

match_extend_back_end_encodeSnappyBetterBlockAsm12B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBetterBlockAsm12B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B

matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm12B

matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B

matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm12B

matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
	JB   match_nolit_end_encodeSnappyBetterBlockAsm12B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeSnappyBetterBlockAsm12B

matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeSnappyBetterBlockAsm12B
	LEAL 1(R12), R12

match_nolit_end_encodeSnappyBetterBlockAsm12B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeSnappyBetterBlockAsm12B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBetterBlockAsm12B
	JB   three_bytes_match_emit_encodeSnappyBetterBlockAsm12B

three_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm12B

two_bytes_match_emit_encodeSnappyBetterBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeSnappyBetterBlockAsm12B
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm12B

one_byte_match_emit_encodeSnappyBetterBlockAsm12B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B

memmove_long_match_emit_encodeSnappyBetterBlockAsm12B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B

two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B

emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBetterBlockAsm12B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x32, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x34, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x32, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x34, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 65536(AX)(R11*4)
	MOVL  R14, 65536(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeSnappyBetterBlockAsm12B:
	CMPQ  R8, R9
	JAE   search_loop_encodeSnappyBetterBlockAsm12B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x32, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x32, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeSnappyBetterBlockAsm12B

emit_remainder_encodeSnappyBetterBlockAsm12B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBetterBlockAsm12B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBetterBlockAsm12B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B
	JB   three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B

three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B

two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBetterBlockAsm12B
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B

one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBetterBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B

memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x000000a0, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBetterBlockAsm10B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBetterBlockAsm10B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBetterBlockAsm10B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBetterBlockAsm10B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x36, R11
	MOVL  (AX)(R10*4), SI
	MOVL  16384(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 16384(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm10B
	CMPQ  R11, DI
	JNE   no_short_found_encodeSnappyBetterBlockAsm10B
	MOVL  R8, SI
	JMP   candidate_match_encodeSnappyBetterBlockAsm10B

no_short_found_encodeSnappyBetterBlockAsm10B:
	CMPL R10, DI
	JEQ  candidate_match_encodeSnappyBetterBlockAsm10B
	CMPL R11, DI
	JEQ  candidateS_match_encodeSnappyBetterBlockAsm10B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBetterBlockAsm10B

candidateS_match_encodeSnappyBetterBlockAsm10B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x34, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm10B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeSnappyBetterBlockAsm10B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBetterBlockAsm10B

match_extend_back_loop_encodeSnappyBetterBlockAsm10B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBetterBlockAsm10B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBetterBlockAsm10B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBetterBlockAsm10B
	JMP  match_extend_back_loop_encodeSnappyBetterBlockAsm10B

match_extend_back_end_encodeSnappyBetterBlockAsm10B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBetterBlockAsm10B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B

matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm10B

matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B

matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm10B

matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
	JB   match_nolit_end_encodeSnappyBetterBlockAsm10B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeSnappyBetterBlockAsm10B

matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeSnappyBetterBlockAsm10B
	LEAL 1(R12), R12

match_nolit_end_encodeSnappyBetterBlockAsm10B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeSnappyBetterBlockAsm10B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBetterBlockAsm10B
	JB   three_bytes_match_emit_encodeSnappyBetterBlockAsm10B

three_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm10B

two_bytes_match_emit_encodeSnappyBetterBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeSnappyBetterBlockAsm10B
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm10B

one_byte_match_emit_encodeSnappyBetterBlockAsm10B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B

memmove_long_match_emit_encodeSnappyBetterBlockAsm10B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B

two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
	CMPL R8, $0x00000800
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B

emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBetterBlockAsm10B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x34, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x36, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x34, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x36, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 16384(AX)(R11*4)
	MOVL  R14, 16384(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeSnappyBetterBlockAsm10B:
	CMPQ  R8, R9
	JAE   search_loop_encodeSnappyBetterBlockAsm10B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x34, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x34, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeSnappyBetterBlockAsm10B

emit_remainder_encodeSnappyBetterBlockAsm10B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBetterBlockAsm10B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBetterBlockAsm10B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B
	JB   three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B

three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B

two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBetterBlockAsm10B
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B

one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBetterBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B

memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int
// Requires: BMI, SSE2
TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64
	MOVQ tmp+48(FP), AX
	MOVQ dst_base+0(FP), CX
	MOVQ $0x00000028, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_encodeSnappyBetterBlockAsm8B:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_encodeSnappyBetterBlockAsm8B
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+32(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  $0x00000000, 16(SP)
	MOVQ  src_base+24(FP), BX

search_loop_encodeSnappyBetterBlockAsm8B:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x04, SI
	LEAL  1(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_encodeSnappyBetterBlockAsm8B
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  $0x9e3779b1, SI
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ SI, R11
	SHRQ  $0x38, R11
	MOVL  (AX)(R10*4), SI
	MOVL  4096(AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	MOVL  DX, 4096(AX)(R11*4)
	MOVQ  (BX)(SI*1), R10
	MOVQ  (BX)(R8*1), R11
	CMPQ  R10, DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm8B
	CMPQ  R11, DI
	JNE   no_short_found_encodeSnappyBetterBlockAsm8B
	MOVL  R8, SI
	JMP   candidate_match_encodeSnappyBetterBlockAsm8B

no_short_found_encodeSnappyBetterBlockAsm8B:
	CMPL R10, DI
	JEQ  candidate_match_encodeSnappyBetterBlockAsm8B
	CMPL R11, DI
	JEQ  candidateS_match_encodeSnappyBetterBlockAsm8B
	MOVL 20(SP), DX
	JMP  search_loop_encodeSnappyBetterBlockAsm8B

candidateS_match_encodeSnappyBetterBlockAsm8B:
	SHRQ  $0x08, DI
	MOVQ  DI, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x36, R10
	MOVL  (AX)(R10*4), SI
	INCL  DX
	MOVL  DX, (AX)(R10*4)
	CMPL  (BX)(SI*1), DI
	JEQ   candidate_match_encodeSnappyBetterBlockAsm8B
	DECL  DX
	MOVL  R8, SI

candidate_match_encodeSnappyBetterBlockAsm8B:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_encodeSnappyBetterBlockAsm8B

match_extend_back_loop_encodeSnappyBetterBlockAsm8B:
	CMPL DX, DI
	JBE  match_extend_back_end_encodeSnappyBetterBlockAsm8B
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_encodeSnappyBetterBlockAsm8B
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_encodeSnappyBetterBlockAsm8B
	JMP  match_extend_back_loop_encodeSnappyBetterBlockAsm8B

match_extend_back_end_encodeSnappyBetterBlockAsm8B:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_encodeSnappyBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_dst_size_check_encodeSnappyBetterBlockAsm8B:
	MOVL DX, DI
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+32(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), R10

	// matchLen
	XORL R12, R12

matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B:
	CMPL R8, $0x10
	JB   matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B
	MOVQ (R9)(R12*1), R11
	MOVQ 8(R9)(R12*1), R13
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
	XORQ 8(R10)(R12*1), R13
	JNZ  matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B
	LEAL -16(R8), R8
	LEAL 16(R12), R12
	JMP  matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B

matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R13, R13

#else
	BSFQ R13, R13

#endif
	SARQ $0x03, R13
	LEAL 8(R12)(R13*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm8B

matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B:
	CMPL R8, $0x08
	JB   matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B
	MOVQ (R9)(R12*1), R11
	XORQ (R10)(R12*1), R11
	JNZ  matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B
	LEAL -8(R8), R8
	LEAL 8(R12), R12
	JMP  matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B

matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL (R12)(R11*1), R12
	JMP  match_nolit_end_encodeSnappyBetterBlockAsm8B

matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B:
	CMPL R8, $0x04
	JB   matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
	MOVL (R9)(R12*1), R11
	CMPL (R10)(R12*1), R11
	JNE  matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B
	LEAL -4(R8), R8
	LEAL 4(R12), R12

matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B:
	CMPL R8, $0x01
	JE   matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
	JB   match_nolit_end_encodeSnappyBetterBlockAsm8B
	MOVW (R9)(R12*1), R11
	CMPW (R10)(R12*1), R11
	JNE  matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B
	LEAL 2(R12), R12
	SUBL $0x02, R8
	JZ   match_nolit_end_encodeSnappyBetterBlockAsm8B

matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B:
	MOVB (R9)(R12*1), R11
	CMPB (R10)(R12*1), R11
	JNE  match_nolit_end_encodeSnappyBetterBlockAsm8B
	LEAL 1(R12), R12

match_nolit_end_encodeSnappyBetterBlockAsm8B:
	MOVL DX, R8
	SUBL SI, R8

	// Check if repeat
	MOVL R8, 16(SP)
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R10
	SUBL SI, R9
	LEAL -1(R9), SI
	CMPL SI, $0x3c
	JB   one_byte_match_emit_encodeSnappyBetterBlockAsm8B
	CMPL SI, $0x00000100
	JB   two_bytes_match_emit_encodeSnappyBetterBlockAsm8B
	JB   three_bytes_match_emit_encodeSnappyBetterBlockAsm8B

three_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW SI, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm8B

two_bytes_match_emit_encodeSnappyBetterBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB SI, 1(CX)
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_match_emit_encodeSnappyBetterBlockAsm8B
	JMP  memmove_long_match_emit_encodeSnappyBetterBlockAsm8B

one_byte_match_emit_encodeSnappyBetterBlockAsm8B:
	SHLB $0x02, SI
	MOVB SI, (CX)
	ADDQ $0x01, CX

memmove_match_emit_encodeSnappyBetterBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8:
	MOVQ (R10), R11
	MOVQ R11, (CX)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
	MOVQ (R10), R11
	MOVQ -8(R10)(R9*1), R10
	MOVQ R11, (CX)
	MOVQ R10, -8(CX)(R9*1)
	JMP  memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
	MOVOU (R10), X0
	MOVOU -16(R10)(R9*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(R9*1)
	JMP   memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)

memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B:
	MOVQ SI, CX
	JMP  emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B

memmove_long_match_emit_encodeSnappyBetterBlockAsm8B:
	LEAQ (CX)(R9*1), SI

	// genMemMoveLong
	MOVOU (R10), X0
	MOVOU 16(R10), X1
	MOVOU -32(R10)(R9*1), X2
	MOVOU -16(R10)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  CX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R14
	SUBQ  R11, R14
	DECQ  R13
	JA    emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(R10)(R14*1), R11
	LEAQ  -32(CX)(R14*1), R15

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R11
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(R10)(R14*1), X4
	MOVOU -16(R10)(R14*1), X5
	MOVOA X4, -32(CX)(R14*1)
	MOVOA X5, -16(CX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(R9*1)
	MOVOU X3, -16(CX)(R9*1)
	MOVQ  SI, CX

emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B:
	ADDL R12, DX
	ADDL $0x04, R12
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B:
	CMPL R12, $0x40
	JBE  two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B
	MOVB $0xee, (CX)
	MOVW R8, 1(CX)
	LEAL -60(R12), R12
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B

two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B:
	MOVL R12, SI
	SHLL $0x02, SI
	CMPL R12, $0x0c
	JAE  emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B
	LEAL -15(SI), SI
	MOVB R8, 1(CX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, SI
	MOVB SI, (CX)
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B

emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B:
	LEAL -2(SI), SI
	MOVB SI, (CX)
	MOVW R8, 1(CX)
	ADDQ $0x03, CX

match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B:
	CMPL DX, 8(SP)
	JAE  emit_remainder_encodeSnappyBetterBlockAsm8B
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B:
	MOVQ  $0x0000cf1bbcdcbf9b, SI
	MOVQ  $0x9e3779b1, R8
	LEAQ  1(DI), DI
	LEAQ  -2(DX), R9
	MOVQ  (BX)(DI*1), R10
	MOVQ  1(BX)(DI*1), R11
	MOVQ  (BX)(R9*1), R12
	MOVQ  1(BX)(R9*1), R13
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x36, R10
	SHLQ  $0x20, R11
	IMULQ R8, R11
	SHRQ  $0x38, R11
	SHLQ  $0x10, R12
	IMULQ SI, R12
	SHRQ  $0x36, R12
	SHLQ  $0x20, R13
	IMULQ R8, R13
	SHRQ  $0x38, R13
	LEAQ  1(DI), R8
	LEAQ  1(R9), R14
	MOVL  DI, (AX)(R10*4)
	MOVL  R9, (AX)(R12*4)
	MOVL  R8, 4096(AX)(R11*4)
	MOVL  R14, 4096(AX)(R13*4)
	LEAQ  1(R9)(DI*1), R8
	SHRQ  $0x01, R8
	ADDQ  $0x01, DI
	SUBQ  $0x01, R9

index_loop_encodeSnappyBetterBlockAsm8B:
	CMPQ  R8, R9
	JAE   search_loop_encodeSnappyBetterBlockAsm8B
	MOVQ  (BX)(DI*1), R10
	MOVQ  (BX)(R8*1), R11
	SHLQ  $0x10, R10
	IMULQ SI, R10
	SHRQ  $0x36, R10
	SHLQ  $0x10, R11
	IMULQ SI, R11
	SHRQ  $0x36, R11
	MOVL  DI, (AX)(R10*4)
	MOVL  R8, (AX)(R11*4)
	ADDQ  $0x02, DI
	ADDQ  $0x02, R8
	JMP   index_loop_encodeSnappyBetterBlockAsm8B

emit_remainder_encodeSnappyBetterBlockAsm8B:
	MOVQ src_len+32(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_encodeSnappyBetterBlockAsm8B
	MOVQ $0x00000000, ret+56(FP)
	RET

emit_remainder_ok_encodeSnappyBetterBlockAsm8B:
	MOVQ src_len+32(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), DX
	CMPL DX, $0x3c
	JB   one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B
	CMPL DX, $0x00000100
	JB   two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B
	JB   three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B

three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
	MOVB $0xf4, (CX)
	MOVW DX, 1(CX)
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B

two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B:
	MOVB $0xf0, (CX)
	MOVB DL, 1(CX)
	ADDQ $0x02, CX
	CMPL DX, $0x40
	JB   memmove_emit_remainder_encodeSnappyBetterBlockAsm8B
	JMP  memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B

one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B:
	SHLB $0x02, DL
	MOVB DL, (CX)
	ADDQ $0x01, CX

memmove_emit_remainder_encodeSnappyBetterBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveShort
	CMPQ BX, $0x03
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2
	JE   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3
	CMPQ BX, $0x08
	JB   emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7
	CMPQ BX, $0x10
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16
	CMPQ BX, $0x20
	JBE  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32
	JMP  emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2:
	MOVB (AX), SI
	MOVB -1(AX)(BX*1), AL
	MOVB SI, (CX)
	MOVB AL, -1(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3:
	MOVW (AX), SI
	MOVB 2(AX), AL
	MOVW SI, (CX)
	MOVB AL, 2(CX)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7:
	MOVL (AX), SI
	MOVL -4(AX)(BX*1), AX
	MOVL SI, (CX)
	MOVL AX, -4(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16:
	MOVQ (AX), SI
	MOVQ -8(AX)(BX*1), AX
	MOVQ SI, (CX)
	MOVQ AX, -8(CX)(BX*1)
	JMP  memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32:
	MOVOU (AX), X0
	MOVOU -16(AX)(BX*1), X1
	MOVOU X0, (CX)
	MOVOU X1, -16(CX)(BX*1)
	JMP   memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B

emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64:
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)

memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B:
	MOVQ DX, CX
	JMP  emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B

memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B:
	LEAQ (CX)(SI*1), DX
	MOVL SI, BX

	// genMemMoveLong
	MOVOU (AX), X0
	MOVOU 16(AX), X1
	MOVOU -32(AX)(BX*1), X2
	MOVOU -16(AX)(BX*1), X3
	MOVQ  BX, DI
	SHRQ  $0x05, DI
	MOVQ  CX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
	LEAQ  -32(AX)(R8*1), SI
	LEAQ  -32(CX)(R8*1), R9

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back

emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32:
	MOVOU -32(AX)(R8*1), X4
	MOVOU -16(AX)(R8*1), X5
	MOVOA X4, -32(CX)(R8*1)
	MOVOA X5, -16(CX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  BX, R8
	JAE   emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_forward_sse_loop_32
	MOVOU X0, (CX)
	MOVOU X1, 16(CX)
	MOVOU X2, -32(CX)(BX*1)
	MOVOU X3, -16(CX)(BX*1)
	MOVQ  DX, CX

emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B:
	MOVQ dst_base+0(FP), AX
	SUBQ AX, CX
	MOVQ CX, ret+56(FP)
	RET

// func calcBlockSize(src []byte, tmp *[32768]byte) int
// Requires: BMI, SSE2
TEXT ·calcBlockSize(SB), $24-40
	MOVQ tmp+24(FP), AX
	XORQ CX, CX
	MOVQ $0x00000100, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_calcBlockSize:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_calcBlockSize
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+8(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+0(FP), BX

search_loop_calcBlockSize:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x05, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_calcBlockSize
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	SHLQ  $0x10, R11
	IMULQ R9, R11
	SHRQ  $0x33, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x10, R10
	IMULQ R9, R10
	SHRQ  $0x33, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_calcBlockSize
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_calcBlockSize

repeat_extend_back_loop_calcBlockSize:
	CMPL DI, SI
	JBE  repeat_extend_back_end_calcBlockSize
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_calcBlockSize
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_calcBlockSize

repeat_extend_back_end_calcBlockSize:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 5(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_calcBlockSize
	MOVQ $0x00000000, ret+32(FP)
	RET

repeat_dst_size_check_calcBlockSize:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_calcBlockSize
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_calcBlockSize
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_calcBlockSize
	CMPL SI, $0x00010000
	JB   three_bytes_repeat_emit_calcBlockSize
	CMPL SI, $0x01000000
	JB   four_bytes_repeat_emit_calcBlockSize
	ADDQ $0x05, CX
	JMP  memmove_long_repeat_emit_calcBlockSize

four_bytes_repeat_emit_calcBlockSize:
	ADDQ $0x04, CX
	JMP  memmove_long_repeat_emit_calcBlockSize

three_bytes_repeat_emit_calcBlockSize:
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_calcBlockSize

two_bytes_repeat_emit_calcBlockSize:
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_calcBlockSize
	JMP  memmove_long_repeat_emit_calcBlockSize

one_byte_repeat_emit_calcBlockSize:
	ADDQ $0x01, CX

memmove_repeat_emit_calcBlockSize:
	LEAQ (CX)(R8*1), CX
	JMP  emit_literal_done_repeat_emit_calcBlockSize

memmove_long_repeat_emit_calcBlockSize:
	LEAQ (CX)(R8*1), CX

emit_literal_done_repeat_emit_calcBlockSize:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+8(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_calcBlockSize:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_calcBlockSize
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_calcBlockSize
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_calcBlockSize
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_calcBlockSize

matchlen_bsf_16repeat_extend_calcBlockSize:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_calcBlockSize

matchlen_match8_repeat_extend_calcBlockSize:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_calcBlockSize
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_calcBlockSize
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_calcBlockSize

matchlen_bsf_8_repeat_extend_calcBlockSize:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_calcBlockSize

matchlen_match4_repeat_extend_calcBlockSize:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_calcBlockSize
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_calcBlockSize
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_calcBlockSize:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_calcBlockSize
	JB   repeat_extend_forward_end_calcBlockSize
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_calcBlockSize
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_calcBlockSize

matchlen_match1_repeat_extend_calcBlockSize:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_calcBlockSize
	LEAL 1(R11), R11

repeat_extend_forward_end_calcBlockSize:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
	CMPL DI, $0x00010000
	JB   two_byte_offset_repeat_as_copy_calcBlockSize

four_bytes_loop_back_repeat_as_copy_calcBlockSize:
	CMPL SI, $0x40
	JBE  four_bytes_remain_repeat_as_copy_calcBlockSize
	LEAL -64(SI), SI
	ADDQ $0x05, CX
	CMPL SI, $0x04
	JB   four_bytes_remain_repeat_as_copy_calcBlockSize
	JMP  four_bytes_loop_back_repeat_as_copy_calcBlockSize

four_bytes_remain_repeat_as_copy_calcBlockSize:
	TESTL SI, SI
	JZ    repeat_end_emit_calcBlockSize
	XORL  SI, SI
	ADDQ  $0x05, CX
	JMP   repeat_end_emit_calcBlockSize

two_byte_offset_repeat_as_copy_calcBlockSize:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_calcBlockSize
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_calcBlockSize

two_byte_offset_short_repeat_as_copy_calcBlockSize:
	MOVL SI, R8
	SHLL $0x02, R8
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_calcBlockSize
	CMPL DI, $0x00000800
	JAE  emit_copy_three_repeat_as_copy_calcBlockSize
	ADDQ $0x02, CX
	JMP  repeat_end_emit_calcBlockSize

emit_copy_three_repeat_as_copy_calcBlockSize:
	ADDQ $0x03, CX

repeat_end_emit_calcBlockSize:
	MOVL DX, 12(SP)
	JMP  search_loop_calcBlockSize

no_repeat_found_calcBlockSize:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_calcBlockSize
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_calcBlockSize
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_calcBlockSize
	MOVL 20(SP), DX
	JMP  search_loop_calcBlockSize

candidate3_match_calcBlockSize:
	ADDL $0x02, DX
	JMP  candidate_match_calcBlockSize

candidate2_match_calcBlockSize:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_calcBlockSize:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_calcBlockSize

match_extend_back_loop_calcBlockSize:
	CMPL DX, DI
	JBE  match_extend_back_end_calcBlockSize
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_calcBlockSize
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_calcBlockSize
	JMP  match_extend_back_loop_calcBlockSize

match_extend_back_end_calcBlockSize:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 5(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_calcBlockSize
	MOVQ $0x00000000, ret+32(FP)
	RET

match_dst_size_check_calcBlockSize:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_calcBlockSize
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), DI
	CMPL DI, $0x3c
	JB   one_byte_match_emit_calcBlockSize
	CMPL DI, $0x00000100
	JB   two_bytes_match_emit_calcBlockSize
	CMPL DI, $0x00010000
	JB   three_bytes_match_emit_calcBlockSize
	CMPL DI, $0x01000000
	JB   four_bytes_match_emit_calcBlockSize
	ADDQ $0x05, CX
	JMP  memmove_long_match_emit_calcBlockSize

four_bytes_match_emit_calcBlockSize:
	ADDQ $0x04, CX
	JMP  memmove_long_match_emit_calcBlockSize

three_bytes_match_emit_calcBlockSize:
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_calcBlockSize

two_bytes_match_emit_calcBlockSize:
	ADDQ $0x02, CX
	CMPL DI, $0x40
	JB   memmove_match_emit_calcBlockSize
	JMP  memmove_long_match_emit_calcBlockSize

one_byte_match_emit_calcBlockSize:
	ADDQ $0x01, CX

memmove_match_emit_calcBlockSize:
	LEAQ (CX)(R9*1), CX
	JMP  emit_literal_done_match_emit_calcBlockSize

memmove_long_match_emit_calcBlockSize:
	LEAQ (CX)(R9*1), CX

emit_literal_done_match_emit_calcBlockSize:
match_nolit_loop_calcBlockSize:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+8(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_calcBlockSize:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_calcBlockSize
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_calcBlockSize
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_calcBlockSize
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_calcBlockSize

matchlen_bsf_16match_nolit_calcBlockSize:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_calcBlockSize

matchlen_match8_match_nolit_calcBlockSize:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_calcBlockSize
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_calcBlockSize
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_calcBlockSize

matchlen_bsf_8_match_nolit_calcBlockSize:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_calcBlockSize

matchlen_match4_match_nolit_calcBlockSize:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_calcBlockSize
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_calcBlockSize
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_calcBlockSize:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_calcBlockSize
	JB   match_nolit_end_calcBlockSize
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_calcBlockSize
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_calcBlockSize

matchlen_match1_match_nolit_calcBlockSize:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_calcBlockSize
	LEAL 1(R10), R10

match_nolit_end_calcBlockSize:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
	CMPL SI, $0x00010000
	JB   two_byte_offset_match_nolit_calcBlockSize

four_bytes_loop_back_match_nolit_calcBlockSize:
	CMPL R10, $0x40
	JBE  four_bytes_remain_match_nolit_calcBlockSize
	LEAL -64(R10), R10
	ADDQ $0x05, CX
	CMPL R10, $0x04
	JB   four_bytes_remain_match_nolit_calcBlockSize
	JMP  four_bytes_loop_back_match_nolit_calcBlockSize

four_bytes_remain_match_nolit_calcBlockSize:
	TESTL R10, R10
	JZ    match_nolit_emitcopy_end_calcBlockSize
	XORL  SI, SI
	ADDQ  $0x05, CX
	JMP   match_nolit_emitcopy_end_calcBlockSize

two_byte_offset_match_nolit_calcBlockSize:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_calcBlockSize
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_calcBlockSize

two_byte_offset_short_match_nolit_calcBlockSize:
	MOVL R10, DI
	SHLL $0x02, DI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_calcBlockSize
	CMPL SI, $0x00000800
	JAE  emit_copy_three_match_nolit_calcBlockSize
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_calcBlockSize

emit_copy_three_match_nolit_calcBlockSize:
	ADDQ $0x03, CX

match_nolit_emitcopy_end_calcBlockSize:
	CMPL DX, 8(SP)
	JAE  emit_remainder_calcBlockSize
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_calcBlockSize
	MOVQ $0x00000000, ret+32(FP)
	RET

match_nolit_dst_ok_calcBlockSize:
	MOVQ  $0x0000cf1bbcdcbf9b, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x10, R8
	IMULQ R9, R8
	SHRQ  $0x33, R8
	SHLQ  $0x10, SI
	IMULQ R9, SI
	SHRQ  $0x33, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_calcBlockSize
	INCL  DX
	JMP   search_loop_calcBlockSize

emit_remainder_calcBlockSize:
	MOVQ src_len+8(FP), AX
	SUBL 12(SP), AX
	LEAQ 5(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_calcBlockSize
	MOVQ $0x00000000, ret+32(FP)
	RET

emit_remainder_ok_calcBlockSize:
	MOVQ src_len+8(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_calcBlockSize
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), AX
	CMPL AX, $0x3c
	JB   one_byte_emit_remainder_calcBlockSize
	CMPL AX, $0x00000100
	JB   two_bytes_emit_remainder_calcBlockSize
	CMPL AX, $0x00010000
	JB   three_bytes_emit_remainder_calcBlockSize
	CMPL AX, $0x01000000
	JB   four_bytes_emit_remainder_calcBlockSize
	ADDQ $0x05, CX
	JMP  memmove_long_emit_remainder_calcBlockSize

four_bytes_emit_remainder_calcBlockSize:
	ADDQ $0x04, CX
	JMP  memmove_long_emit_remainder_calcBlockSize

three_bytes_emit_remainder_calcBlockSize:
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_calcBlockSize

two_bytes_emit_remainder_calcBlockSize:
	ADDQ $0x02, CX
	CMPL AX, $0x40
	JB   memmove_emit_remainder_calcBlockSize
	JMP  memmove_long_emit_remainder_calcBlockSize

one_byte_emit_remainder_calcBlockSize:
	ADDQ $0x01, CX

memmove_emit_remainder_calcBlockSize:
	LEAQ (CX)(SI*1), AX
	MOVQ AX, CX
	JMP  emit_literal_done_emit_remainder_calcBlockSize

memmove_long_emit_remainder_calcBlockSize:
	LEAQ (CX)(SI*1), AX
	MOVQ AX, CX

emit_literal_done_emit_remainder_calcBlockSize:
	MOVQ CX, ret+32(FP)
	RET

// func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int
// Requires: BMI, SSE2
TEXT ·calcBlockSizeSmall(SB), $24-40
	MOVQ tmp+24(FP), AX
	XORQ CX, CX
	MOVQ $0x00000010, DX
	MOVQ AX, BX
	PXOR X0, X0

zero_loop_calcBlockSizeSmall:
	MOVOU X0, (BX)
	MOVOU X0, 16(BX)
	MOVOU X0, 32(BX)
	MOVOU X0, 48(BX)
	MOVOU X0, 64(BX)
	MOVOU X0, 80(BX)
	MOVOU X0, 96(BX)
	MOVOU X0, 112(BX)
	ADDQ  $0x80, BX
	DECQ  DX
	JNZ   zero_loop_calcBlockSizeSmall
	MOVL  $0x00000000, 12(SP)
	MOVQ  src_len+8(FP), DX
	LEAQ  -9(DX), BX
	LEAQ  -8(DX), SI
	MOVL  SI, 8(SP)
	SHRQ  $0x05, DX
	SUBL  DX, BX
	LEAQ  (CX)(BX*1), BX
	MOVQ  BX, (SP)
	MOVL  $0x00000001, DX
	MOVL  DX, 16(SP)
	MOVQ  src_base+0(FP), BX

search_loop_calcBlockSizeSmall:
	MOVL  DX, SI
	SUBL  12(SP), SI
	SHRL  $0x04, SI
	LEAL  4(DX)(SI*1), SI
	CMPL  SI, 8(SP)
	JAE   emit_remainder_calcBlockSizeSmall
	MOVQ  (BX)(DX*1), DI
	MOVL  SI, 20(SP)
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R10
	MOVQ  DI, R11
	SHRQ  $0x08, R11
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x37, R10
	SHLQ  $0x20, R11
	IMULQ R9, R11
	SHRQ  $0x37, R11
	MOVL  (AX)(R10*4), SI
	MOVL  (AX)(R11*4), R8
	MOVL  DX, (AX)(R10*4)
	LEAL  1(DX), R10
	MOVL  R10, (AX)(R11*4)
	MOVQ  DI, R10
	SHRQ  $0x10, R10
	SHLQ  $0x20, R10
	IMULQ R9, R10
	SHRQ  $0x37, R10
	MOVL  DX, R9
	SUBL  16(SP), R9
	MOVL  1(BX)(R9*1), R11
	MOVQ  DI, R9
	SHRQ  $0x08, R9
	CMPL  R9, R11
	JNE   no_repeat_found_calcBlockSizeSmall
	LEAL  1(DX), DI
	MOVL  12(SP), SI
	MOVL  DI, R8
	SUBL  16(SP), R8
	JZ    repeat_extend_back_end_calcBlockSizeSmall

repeat_extend_back_loop_calcBlockSizeSmall:
	CMPL DI, SI
	JBE  repeat_extend_back_end_calcBlockSizeSmall
	MOVB -1(BX)(R8*1), R9
	MOVB -1(BX)(DI*1), R10
	CMPB R9, R10
	JNE  repeat_extend_back_end_calcBlockSizeSmall
	LEAL -1(DI), DI
	DECL R8
	JNZ  repeat_extend_back_loop_calcBlockSizeSmall

repeat_extend_back_end_calcBlockSizeSmall:
	MOVL DI, SI
	SUBL 12(SP), SI
	LEAQ 3(CX)(SI*1), SI
	CMPQ SI, (SP)
	JB   repeat_dst_size_check_calcBlockSizeSmall
	MOVQ $0x00000000, ret+32(FP)
	RET

repeat_dst_size_check_calcBlockSizeSmall:
	MOVL 12(SP), SI
	CMPL SI, DI
	JEQ  emit_literal_done_repeat_emit_calcBlockSizeSmall
	MOVL DI, R8
	MOVL DI, 12(SP)
	LEAQ (BX)(SI*1), R9
	SUBL SI, R8
	LEAL -1(R8), SI
	CMPL SI, $0x3c
	JB   one_byte_repeat_emit_calcBlockSizeSmall
	CMPL SI, $0x00000100
	JB   two_bytes_repeat_emit_calcBlockSizeSmall
	JB   three_bytes_repeat_emit_calcBlockSizeSmall

three_bytes_repeat_emit_calcBlockSizeSmall:
	ADDQ $0x03, CX
	JMP  memmove_long_repeat_emit_calcBlockSizeSmall

two_bytes_repeat_emit_calcBlockSizeSmall:
	ADDQ $0x02, CX
	CMPL SI, $0x40
	JB   memmove_repeat_emit_calcBlockSizeSmall
	JMP  memmove_long_repeat_emit_calcBlockSizeSmall

one_byte_repeat_emit_calcBlockSizeSmall:
	ADDQ $0x01, CX

memmove_repeat_emit_calcBlockSizeSmall:
	LEAQ (CX)(R8*1), CX
	JMP  emit_literal_done_repeat_emit_calcBlockSizeSmall

memmove_long_repeat_emit_calcBlockSizeSmall:
	LEAQ (CX)(R8*1), CX

emit_literal_done_repeat_emit_calcBlockSizeSmall:
	ADDL $0x05, DX
	MOVL DX, SI
	SUBL 16(SP), SI
	MOVQ src_len+8(FP), R8
	SUBL DX, R8
	LEAQ (BX)(DX*1), R9
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R11, R11

matchlen_loopback_16_repeat_extend_calcBlockSizeSmall:
	CMPL R8, $0x10
	JB   matchlen_match8_repeat_extend_calcBlockSizeSmall
	MOVQ (R9)(R11*1), R10
	MOVQ 8(R9)(R11*1), R12
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
	XORQ 8(SI)(R11*1), R12
	JNZ  matchlen_bsf_16repeat_extend_calcBlockSizeSmall
	LEAL -16(R8), R8
	LEAL 16(R11), R11
	JMP  matchlen_loopback_16_repeat_extend_calcBlockSizeSmall

matchlen_bsf_16repeat_extend_calcBlockSizeSmall:
#ifdef GOAMD64_v3
	TZCNTQ R12, R12

#else
	BSFQ R12, R12

#endif
	SARQ $0x03, R12
	LEAL 8(R11)(R12*1), R11
	JMP  repeat_extend_forward_end_calcBlockSizeSmall

matchlen_match8_repeat_extend_calcBlockSizeSmall:
	CMPL R8, $0x08
	JB   matchlen_match4_repeat_extend_calcBlockSizeSmall
	MOVQ (R9)(R11*1), R10
	XORQ (SI)(R11*1), R10
	JNZ  matchlen_bsf_8_repeat_extend_calcBlockSizeSmall
	LEAL -8(R8), R8
	LEAL 8(R11), R11
	JMP  matchlen_match4_repeat_extend_calcBlockSizeSmall

matchlen_bsf_8_repeat_extend_calcBlockSizeSmall:
#ifdef GOAMD64_v3
	TZCNTQ R10, R10

#else
	BSFQ R10, R10

#endif
	SARQ $0x03, R10
	LEAL (R11)(R10*1), R11
	JMP  repeat_extend_forward_end_calcBlockSizeSmall

matchlen_match4_repeat_extend_calcBlockSizeSmall:
	CMPL R8, $0x04
	JB   matchlen_match2_repeat_extend_calcBlockSizeSmall
	MOVL (R9)(R11*1), R10
	CMPL (SI)(R11*1), R10
	JNE  matchlen_match2_repeat_extend_calcBlockSizeSmall
	LEAL -4(R8), R8
	LEAL 4(R11), R11

matchlen_match2_repeat_extend_calcBlockSizeSmall:
	CMPL R8, $0x01
	JE   matchlen_match1_repeat_extend_calcBlockSizeSmall
	JB   repeat_extend_forward_end_calcBlockSizeSmall
	MOVW (R9)(R11*1), R10
	CMPW (SI)(R11*1), R10
	JNE  matchlen_match1_repeat_extend_calcBlockSizeSmall
	LEAL 2(R11), R11
	SUBL $0x02, R8
	JZ   repeat_extend_forward_end_calcBlockSizeSmall

matchlen_match1_repeat_extend_calcBlockSizeSmall:
	MOVB (R9)(R11*1), R10
	CMPB (SI)(R11*1), R10
	JNE  repeat_extend_forward_end_calcBlockSizeSmall
	LEAL 1(R11), R11

repeat_extend_forward_end_calcBlockSizeSmall:
	ADDL R11, DX
	MOVL DX, SI
	SUBL DI, SI
	MOVL 16(SP), DI

	// emitCopy
two_byte_offset_repeat_as_copy_calcBlockSizeSmall:
	CMPL SI, $0x40
	JBE  two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall
	LEAL -60(SI), SI
	ADDQ $0x03, CX
	JMP  two_byte_offset_repeat_as_copy_calcBlockSizeSmall

two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall:
	MOVL SI, DI
	SHLL $0x02, DI
	CMPL SI, $0x0c
	JAE  emit_copy_three_repeat_as_copy_calcBlockSizeSmall
	ADDQ $0x02, CX
	JMP  repeat_end_emit_calcBlockSizeSmall

emit_copy_three_repeat_as_copy_calcBlockSizeSmall:
	ADDQ $0x03, CX

repeat_end_emit_calcBlockSizeSmall:
	MOVL DX, 12(SP)
	JMP  search_loop_calcBlockSizeSmall

no_repeat_found_calcBlockSizeSmall:
	CMPL (BX)(SI*1), DI
	JEQ  candidate_match_calcBlockSizeSmall
	SHRQ $0x08, DI
	MOVL (AX)(R10*4), SI
	LEAL 2(DX), R9
	CMPL (BX)(R8*1), DI
	JEQ  candidate2_match_calcBlockSizeSmall
	MOVL R9, (AX)(R10*4)
	SHRQ $0x08, DI
	CMPL (BX)(SI*1), DI
	JEQ  candidate3_match_calcBlockSizeSmall
	MOVL 20(SP), DX
	JMP  search_loop_calcBlockSizeSmall

candidate3_match_calcBlockSizeSmall:
	ADDL $0x02, DX
	JMP  candidate_match_calcBlockSizeSmall

candidate2_match_calcBlockSizeSmall:
	MOVL R9, (AX)(R10*4)
	INCL DX
	MOVL R8, SI

candidate_match_calcBlockSizeSmall:
	MOVL  12(SP), DI
	TESTL SI, SI
	JZ    match_extend_back_end_calcBlockSizeSmall

match_extend_back_loop_calcBlockSizeSmall:
	CMPL DX, DI
	JBE  match_extend_back_end_calcBlockSizeSmall
	MOVB -1(BX)(SI*1), R8
	MOVB -1(BX)(DX*1), R9
	CMPB R8, R9
	JNE  match_extend_back_end_calcBlockSizeSmall
	LEAL -1(DX), DX
	DECL SI
	JZ   match_extend_back_end_calcBlockSizeSmall
	JMP  match_extend_back_loop_calcBlockSizeSmall

match_extend_back_end_calcBlockSizeSmall:
	MOVL DX, DI
	SUBL 12(SP), DI
	LEAQ 3(CX)(DI*1), DI
	CMPQ DI, (SP)
	JB   match_dst_size_check_calcBlockSizeSmall
	MOVQ $0x00000000, ret+32(FP)
	RET

match_dst_size_check_calcBlockSizeSmall:
	MOVL DX, DI
	MOVL 12(SP), R8
	CMPL R8, DI
	JEQ  emit_literal_done_match_emit_calcBlockSizeSmall
	MOVL DI, R9
	MOVL DI, 12(SP)
	LEAQ (BX)(R8*1), DI
	SUBL R8, R9
	LEAL -1(R9), DI
	CMPL DI, $0x3c
	JB   one_byte_match_emit_calcBlockSizeSmall
	CMPL DI, $0x00000100
	JB   two_bytes_match_emit_calcBlockSizeSmall
	JB   three_bytes_match_emit_calcBlockSizeSmall

three_bytes_match_emit_calcBlockSizeSmall:
	ADDQ $0x03, CX
	JMP  memmove_long_match_emit_calcBlockSizeSmall

two_bytes_match_emit_calcBlockSizeSmall:
	ADDQ $0x02, CX
	CMPL DI, $0x40
	JB   memmove_match_emit_calcBlockSizeSmall
	JMP  memmove_long_match_emit_calcBlockSizeSmall

one_byte_match_emit_calcBlockSizeSmall:
	ADDQ $0x01, CX

memmove_match_emit_calcBlockSizeSmall:
	LEAQ (CX)(R9*1), CX
	JMP  emit_literal_done_match_emit_calcBlockSizeSmall

memmove_long_match_emit_calcBlockSizeSmall:
	LEAQ (CX)(R9*1), CX

emit_literal_done_match_emit_calcBlockSizeSmall:
match_nolit_loop_calcBlockSizeSmall:
	MOVL DX, DI
	SUBL SI, DI
	MOVL DI, 16(SP)
	ADDL $0x04, DX
	ADDL $0x04, SI
	MOVQ src_len+8(FP), DI
	SUBL DX, DI
	LEAQ (BX)(DX*1), R8
	LEAQ (BX)(SI*1), SI

	// matchLen
	XORL R10, R10

matchlen_loopback_16_match_nolit_calcBlockSizeSmall:
	CMPL DI, $0x10
	JB   matchlen_match8_match_nolit_calcBlockSizeSmall
	MOVQ (R8)(R10*1), R9
	MOVQ 8(R8)(R10*1), R11
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_calcBlockSizeSmall
	XORQ 8(SI)(R10*1), R11
	JNZ  matchlen_bsf_16match_nolit_calcBlockSizeSmall
	LEAL -16(DI), DI
	LEAL 16(R10), R10
	JMP  matchlen_loopback_16_match_nolit_calcBlockSizeSmall

matchlen_bsf_16match_nolit_calcBlockSizeSmall:
#ifdef GOAMD64_v3
	TZCNTQ R11, R11

#else
	BSFQ R11, R11

#endif
	SARQ $0x03, R11
	LEAL 8(R10)(R11*1), R10
	JMP  match_nolit_end_calcBlockSizeSmall

matchlen_match8_match_nolit_calcBlockSizeSmall:
	CMPL DI, $0x08
	JB   matchlen_match4_match_nolit_calcBlockSizeSmall
	MOVQ (R8)(R10*1), R9
	XORQ (SI)(R10*1), R9
	JNZ  matchlen_bsf_8_match_nolit_calcBlockSizeSmall
	LEAL -8(DI), DI
	LEAL 8(R10), R10
	JMP  matchlen_match4_match_nolit_calcBlockSizeSmall

matchlen_bsf_8_match_nolit_calcBlockSizeSmall:
#ifdef GOAMD64_v3
	TZCNTQ R9, R9

#else
	BSFQ R9, R9

#endif
	SARQ $0x03, R9
	LEAL (R10)(R9*1), R10
	JMP  match_nolit_end_calcBlockSizeSmall

matchlen_match4_match_nolit_calcBlockSizeSmall:
	CMPL DI, $0x04
	JB   matchlen_match2_match_nolit_calcBlockSizeSmall
	MOVL (R8)(R10*1), R9
	CMPL (SI)(R10*1), R9
	JNE  matchlen_match2_match_nolit_calcBlockSizeSmall
	LEAL -4(DI), DI
	LEAL 4(R10), R10

matchlen_match2_match_nolit_calcBlockSizeSmall:
	CMPL DI, $0x01
	JE   matchlen_match1_match_nolit_calcBlockSizeSmall
	JB   match_nolit_end_calcBlockSizeSmall
	MOVW (R8)(R10*1), R9
	CMPW (SI)(R10*1), R9
	JNE  matchlen_match1_match_nolit_calcBlockSizeSmall
	LEAL 2(R10), R10
	SUBL $0x02, DI
	JZ   match_nolit_end_calcBlockSizeSmall

matchlen_match1_match_nolit_calcBlockSizeSmall:
	MOVB (R8)(R10*1), R9
	CMPB (SI)(R10*1), R9
	JNE  match_nolit_end_calcBlockSizeSmall
	LEAL 1(R10), R10

match_nolit_end_calcBlockSizeSmall:
	ADDL R10, DX
	MOVL 16(SP), SI
	ADDL $0x04, R10
	MOVL DX, 12(SP)

	// emitCopy
two_byte_offset_match_nolit_calcBlockSizeSmall:
	CMPL R10, $0x40
	JBE  two_byte_offset_short_match_nolit_calcBlockSizeSmall
	LEAL -60(R10), R10
	ADDQ $0x03, CX
	JMP  two_byte_offset_match_nolit_calcBlockSizeSmall

two_byte_offset_short_match_nolit_calcBlockSizeSmall:
	MOVL R10, SI
	SHLL $0x02, SI
	CMPL R10, $0x0c
	JAE  emit_copy_three_match_nolit_calcBlockSizeSmall
	ADDQ $0x02, CX
	JMP  match_nolit_emitcopy_end_calcBlockSizeSmall

emit_copy_three_match_nolit_calcBlockSizeSmall:
	ADDQ $0x03, CX

match_nolit_emitcopy_end_calcBlockSizeSmall:
	CMPL DX, 8(SP)
	JAE  emit_remainder_calcBlockSizeSmall
	MOVQ -2(BX)(DX*1), DI
	CMPQ CX, (SP)
	JB   match_nolit_dst_ok_calcBlockSizeSmall
	MOVQ $0x00000000, ret+32(FP)
	RET

match_nolit_dst_ok_calcBlockSizeSmall:
	MOVQ  $0x9e3779b1, R9
	MOVQ  DI, R8
	SHRQ  $0x10, DI
	MOVQ  DI, SI
	SHLQ  $0x20, R8
	IMULQ R9, R8
	SHRQ  $0x37, R8
	SHLQ  $0x20, SI
	IMULQ R9, SI
	SHRQ  $0x37, SI
	LEAL  -2(DX), R9
	LEAQ  (AX)(SI*4), R10
	MOVL  (R10), SI
	MOVL  R9, (AX)(R8*4)
	MOVL  DX, (R10)
	CMPL  (BX)(SI*1), DI
	JEQ   match_nolit_loop_calcBlockSizeSmall
	INCL  DX
	JMP   search_loop_calcBlockSizeSmall

emit_remainder_calcBlockSizeSmall:
	MOVQ src_len+8(FP), AX
	SUBL 12(SP), AX
	LEAQ 3(CX)(AX*1), AX
	CMPQ AX, (SP)
	JB   emit_remainder_ok_calcBlockSizeSmall
	MOVQ $0x00000000, ret+32(FP)
	RET

emit_remainder_ok_calcBlockSizeSmall:
	MOVQ src_len+8(FP), AX
	MOVL 12(SP), DX
	CMPL DX, AX
	JEQ  emit_literal_done_emit_remainder_calcBlockSizeSmall
	MOVL AX, SI
	MOVL AX, 12(SP)
	LEAQ (BX)(DX*1), AX
	SUBL DX, SI
	LEAL -1(SI), AX
	CMPL AX, $0x3c
	JB   one_byte_emit_remainder_calcBlockSizeSmall
	CMPL AX, $0x00000100
	JB   two_bytes_emit_remainder_calcBlockSizeSmall
	JB   three_bytes_emit_remainder_calcBlockSizeSmall

three_bytes_emit_remainder_calcBlockSizeSmall:
	ADDQ $0x03, CX
	JMP  memmove_long_emit_remainder_calcBlockSizeSmall

two_bytes_emit_remainder_calcBlockSizeSmall:
	ADDQ $0x02, CX
	CMPL AX, $0x40
	JB   memmove_emit_remainder_calcBlockSizeSmall
	JMP  memmove_long_emit_remainder_calcBlockSizeSmall

one_byte_emit_remainder_calcBlockSizeSmall:
	ADDQ $0x01, CX

memmove_emit_remainder_calcBlockSizeSmall:
	LEAQ (CX)(SI*1), AX
	MOVQ AX, CX
	JMP  emit_literal_done_emit_remainder_calcBlockSizeSmall

memmove_long_emit_remainder_calcBlockSizeSmall:
	LEAQ (CX)(SI*1), AX
	MOVQ AX, CX

emit_literal_done_emit_remainder_calcBlockSizeSmall:
	MOVQ CX, ret+32(FP)
	RET

// func emitLiteral(dst []byte, lit []byte) int
// Requires: SSE2
TEXT ·emitLiteral(SB), NOSPLIT, $0-56
	MOVQ  lit_len+32(FP), DX
	MOVQ  dst_base+0(FP), AX
	MOVQ  lit_base+24(FP), CX
	TESTQ DX, DX
	JZ    emit_literal_end_standalone_skip
	MOVL  DX, BX
	LEAL  -1(DX), SI
	CMPL  SI, $0x3c
	JB    one_byte_standalone
	CMPL  SI, $0x00000100
	JB    two_bytes_standalone
	CMPL  SI, $0x00010000
	JB    three_bytes_standalone
	CMPL  SI, $0x01000000
	JB    four_bytes_standalone
	MOVB  $0xfc, (AX)
	MOVL  SI, 1(AX)
	ADDQ  $0x05, BX
	ADDQ  $0x05, AX
	JMP   memmove_long_standalone

four_bytes_standalone:
	MOVL SI, DI
	SHRL $0x10, DI
	MOVB $0xf8, (AX)
	MOVW SI, 1(AX)
	MOVB DI, 3(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  memmove_long_standalone

three_bytes_standalone:
	MOVB $0xf4, (AX)
	MOVW SI, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  memmove_long_standalone

two_bytes_standalone:
	MOVB $0xf0, (AX)
	MOVB SI, 1(AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	CMPL SI, $0x40
	JB   memmove_standalone
	JMP  memmove_long_standalone

one_byte_standalone:
	SHLB $0x02, SI
	MOVB SI, (AX)
	ADDQ $0x01, BX
	ADDQ $0x01, AX

memmove_standalone:
	// genMemMoveShort
	CMPQ DX, $0x03
	JB   emit_lit_memmove_standalone_memmove_move_1or2
	JE   emit_lit_memmove_standalone_memmove_move_3
	CMPQ DX, $0x08
	JB   emit_lit_memmove_standalone_memmove_move_4through7
	CMPQ DX, $0x10
	JBE  emit_lit_memmove_standalone_memmove_move_8through16
	CMPQ DX, $0x20
	JBE  emit_lit_memmove_standalone_memmove_move_17through32
	JMP  emit_lit_memmove_standalone_memmove_move_33through64

emit_lit_memmove_standalone_memmove_move_1or2:
	MOVB (CX), SI
	MOVB -1(CX)(DX*1), CL
	MOVB SI, (AX)
	MOVB CL, -1(AX)(DX*1)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_3:
	MOVW (CX), SI
	MOVB 2(CX), CL
	MOVW SI, (AX)
	MOVB CL, 2(AX)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_4through7:
	MOVL (CX), SI
	MOVL -4(CX)(DX*1), CX
	MOVL SI, (AX)
	MOVL CX, -4(AX)(DX*1)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_8through16:
	MOVQ (CX), SI
	MOVQ -8(CX)(DX*1), CX
	MOVQ SI, (AX)
	MOVQ CX, -8(AX)(DX*1)
	JMP  emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_17through32:
	MOVOU (CX), X0
	MOVOU -16(CX)(DX*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone

emit_lit_memmove_standalone_memmove_move_33through64:
	MOVOU (CX), X0
	MOVOU 16(CX), X1
	MOVOU -32(CX)(DX*1), X2
	MOVOU -16(CX)(DX*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone
	JMP emit_literal_end_standalone

memmove_long_standalone:
	// genMemMoveLong
	MOVOU (CX), X0
	MOVOU 16(CX), X1
	MOVOU -32(CX)(DX*1), X2
	MOVOU -16(CX)(DX*1), X3
	MOVQ  DX, DI
	SHRQ  $0x05, DI
	MOVQ  AX, SI
	ANDL  $0x0000001f, SI
	MOVQ  $0x00000040, R8
	SUBQ  SI, R8
	DECQ  DI
	JA    emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
	LEAQ  -32(CX)(R8*1), SI
	LEAQ  -32(AX)(R8*1), R9

emit_lit_memmove_long_standalonelarge_big_loop_back:
	MOVOU (SI), X4
	MOVOU 16(SI), X5
	MOVOA X4, (R9)
	MOVOA X5, 16(R9)
	ADDQ  $0x20, R9
	ADDQ  $0x20, SI
	ADDQ  $0x20, R8
	DECQ  DI
	JNA   emit_lit_memmove_long_standalonelarge_big_loop_back

emit_lit_memmove_long_standalonelarge_forward_sse_loop_32:
	MOVOU -32(CX)(R8*1), X4
	MOVOU -16(CX)(R8*1), X5
	MOVOA X4, -32(AX)(R8*1)
	MOVOA X5, -16(AX)(R8*1)
	ADDQ  $0x20, R8
	CMPQ  DX, R8
	JAE   emit_lit_memmove_long_standalonelarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(DX*1)
	MOVOU X3, -16(AX)(DX*1)
	JMP   emit_literal_end_standalone
	JMP emit_literal_end_standalone

emit_literal_end_standalone_skip:
	XORQ BX, BX

emit_literal_end_standalone:
	MOVQ BX, ret+48(FP)
	RET

// func emitRepeat(dst []byte, offset int, length int) int
TEXT ·emitRepeat(SB), NOSPLIT, $0-48
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ offset+24(FP), CX
	MOVQ length+32(FP), DX

	// emitRepeat
emit_repeat_again_standalone:
	MOVL DX, SI
	LEAL -4(DX), DX
	CMPL SI, $0x08
	JBE  repeat_two_standalone
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_standalone
	CMPL CX, $0x00000800
	JB   repeat_two_offset_standalone

cant_repeat_two_offset_standalone:
	CMPL DX, $0x00000104
	JB   repeat_three_standalone
	CMPL DX, $0x00010100
	JB   repeat_four_standalone
	CMPL DX, $0x0100ffff
	JB   repeat_five_standalone
	LEAL -16842747(DX), DX
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	ADDQ $0x05, BX
	JMP  emit_repeat_again_standalone

repeat_five_standalone:
	LEAL -65536(DX), DX
	MOVL DX, CX
	MOVW $0x001d, (AX)
	MOVW DX, 2(AX)
	SARL $0x10, CX
	MOVB CL, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_repeat_end

repeat_four_standalone:
	LEAL -256(DX), DX
	MOVW $0x0019, (AX)
	MOVW DX, 2(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_repeat_end

repeat_three_standalone:
	LEAL -4(DX), DX
	MOVW $0x0015, (AX)
	MOVB DL, 2(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_repeat_end

repeat_two_standalone:
	SHLL $0x02, DX
	ORL  $0x01, DX
	MOVW DX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_repeat_end

repeat_two_offset_standalone:
	XORQ SI, SI
	LEAL 1(SI)(DX*4), DX
	MOVB CL, 1(AX)
	SARL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, DX
	MOVB DL, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX

gen_emit_repeat_end:
	MOVQ BX, ret+40(FP)
	RET

// func emitCopy(dst []byte, offset int, length int) int
TEXT ·emitCopy(SB), NOSPLIT, $0-48
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ offset+24(FP), CX
	MOVQ length+32(FP), DX

	// emitCopy
	CMPL CX, $0x00010000
	JB   two_byte_offset_standalone
	CMPL DX, $0x40
	JBE  four_bytes_remain_standalone
	MOVB $0xff, (AX)
	MOVL CX, 1(AX)
	LEAL -64(DX), DX
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	CMPL DX, $0x04
	JB   four_bytes_remain_standalone

	// emitRepeat
emit_repeat_again_standalone_emit_copy:
	MOVL DX, SI
	LEAL -4(DX), DX
	CMPL SI, $0x08
	JBE  repeat_two_standalone_emit_copy
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_standalone_emit_copy
	CMPL CX, $0x00000800
	JB   repeat_two_offset_standalone_emit_copy

cant_repeat_two_offset_standalone_emit_copy:
	CMPL DX, $0x00000104
	JB   repeat_three_standalone_emit_copy
	CMPL DX, $0x00010100
	JB   repeat_four_standalone_emit_copy
	CMPL DX, $0x0100ffff
	JB   repeat_five_standalone_emit_copy
	LEAL -16842747(DX), DX
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	ADDQ $0x05, BX
	JMP  emit_repeat_again_standalone_emit_copy

repeat_five_standalone_emit_copy:
	LEAL -65536(DX), DX
	MOVL DX, CX
	MOVW $0x001d, (AX)
	MOVW DX, 2(AX)
	SARL $0x10, CX
	MOVB CL, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_end

repeat_four_standalone_emit_copy:
	LEAL -256(DX), DX
	MOVW $0x0019, (AX)
	MOVW DX, 2(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

repeat_three_standalone_emit_copy:
	LEAL -4(DX), DX
	MOVW $0x0015, (AX)
	MOVB DL, 2(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_copy_end

repeat_two_standalone_emit_copy:
	SHLL $0x02, DX
	ORL  $0x01, DX
	MOVW DX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

repeat_two_offset_standalone_emit_copy:
	XORQ SI, SI
	LEAL 1(SI)(DX*4), DX
	MOVB CL, 1(AX)
	SARL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, DX
	MOVB DL, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

four_bytes_remain_standalone:
	TESTL DX, DX
	JZ    gen_emit_copy_end
	XORL  SI, SI
	LEAL  -1(SI)(DX*4), DX
	MOVB  DL, (AX)
	MOVL  CX, 1(AX)
	ADDQ  $0x05, BX
	ADDQ  $0x05, AX
	JMP   gen_emit_copy_end

two_byte_offset_standalone:
	CMPL DX, $0x40
	JBE  two_byte_offset_short_standalone
	CMPL CX, $0x00000800
	JAE  long_offset_short_standalone
	MOVL $0x00000001, SI
	LEAL 16(SI), SI
	MOVB CL, 1(AX)
	MOVL CX, DI
	SHRL $0x08, DI
	SHLL $0x05, DI
	ORL  DI, SI
	MOVB SI, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	SUBL $0x08, DX

	// emitRepeat
	LEAL -4(DX), DX
	JMP  cant_repeat_two_offset_standalone_emit_copy_short_2b

emit_repeat_again_standalone_emit_copy_short_2b:
	MOVL DX, SI
	LEAL -4(DX), DX
	CMPL SI, $0x08
	JBE  repeat_two_standalone_emit_copy_short_2b
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_standalone_emit_copy_short_2b
	CMPL CX, $0x00000800
	JB   repeat_two_offset_standalone_emit_copy_short_2b

cant_repeat_two_offset_standalone_emit_copy_short_2b:
	CMPL DX, $0x00000104
	JB   repeat_three_standalone_emit_copy_short_2b
	CMPL DX, $0x00010100
	JB   repeat_four_standalone_emit_copy_short_2b
	CMPL DX, $0x0100ffff
	JB   repeat_five_standalone_emit_copy_short_2b
	LEAL -16842747(DX), DX
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	ADDQ $0x05, BX
	JMP  emit_repeat_again_standalone_emit_copy_short_2b

repeat_five_standalone_emit_copy_short_2b:
	LEAL -65536(DX), DX
	MOVL DX, CX
	MOVW $0x001d, (AX)
	MOVW DX, 2(AX)
	SARL $0x10, CX
	MOVB CL, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_end

repeat_four_standalone_emit_copy_short_2b:
	LEAL -256(DX), DX
	MOVW $0x0019, (AX)
	MOVW DX, 2(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

repeat_three_standalone_emit_copy_short_2b:
	LEAL -4(DX), DX
	MOVW $0x0015, (AX)
	MOVB DL, 2(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_copy_end

repeat_two_standalone_emit_copy_short_2b:
	SHLL $0x02, DX
	ORL  $0x01, DX
	MOVW DX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

repeat_two_offset_standalone_emit_copy_short_2b:
	XORQ SI, SI
	LEAL 1(SI)(DX*4), DX
	MOVB CL, 1(AX)
	SARL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, DX
	MOVB DL, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

long_offset_short_standalone:
	MOVB $0xee, (AX)
	MOVW CX, 1(AX)
	LEAL -60(DX), DX
	ADDQ $0x03, AX
	ADDQ $0x03, BX

	// emitRepeat
emit_repeat_again_standalone_emit_copy_short:
	MOVL DX, SI
	LEAL -4(DX), DX
	CMPL SI, $0x08
	JBE  repeat_two_standalone_emit_copy_short
	CMPL SI, $0x0c
	JAE  cant_repeat_two_offset_standalone_emit_copy_short
	CMPL CX, $0x00000800
	JB   repeat_two_offset_standalone_emit_copy_short

cant_repeat_two_offset_standalone_emit_copy_short:
	CMPL DX, $0x00000104
	JB   repeat_three_standalone_emit_copy_short
	CMPL DX, $0x00010100
	JB   repeat_four_standalone_emit_copy_short
	CMPL DX, $0x0100ffff
	JB   repeat_five_standalone_emit_copy_short
	LEAL -16842747(DX), DX
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	ADDQ $0x05, BX
	JMP  emit_repeat_again_standalone_emit_copy_short

repeat_five_standalone_emit_copy_short:
	LEAL -65536(DX), DX
	MOVL DX, CX
	MOVW $0x001d, (AX)
	MOVW DX, 2(AX)
	SARL $0x10, CX
	MOVB CL, 4(AX)
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	JMP  gen_emit_copy_end

repeat_four_standalone_emit_copy_short:
	LEAL -256(DX), DX
	MOVW $0x0019, (AX)
	MOVW DX, 2(AX)
	ADDQ $0x04, BX
	ADDQ $0x04, AX
	JMP  gen_emit_copy_end

repeat_three_standalone_emit_copy_short:
	LEAL -4(DX), DX
	MOVW $0x0015, (AX)
	MOVB DL, 2(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX
	JMP  gen_emit_copy_end

repeat_two_standalone_emit_copy_short:
	SHLL $0x02, DX
	ORL  $0x01, DX
	MOVW DX, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

repeat_two_offset_standalone_emit_copy_short:
	XORQ SI, SI
	LEAL 1(SI)(DX*4), DX
	MOVB CL, 1(AX)
	SARL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, DX
	MOVB DL, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

two_byte_offset_short_standalone:
	MOVL DX, SI
	SHLL $0x02, SI
	CMPL DX, $0x0c
	JAE  emit_copy_three_standalone
	CMPL CX, $0x00000800
	JAE  emit_copy_three_standalone
	LEAL -15(SI), SI
	MOVB CL, 1(AX)
	SHRL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, SI
	MOVB SI, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end

emit_copy_three_standalone:
	LEAL -2(SI), SI
	MOVB SI, (AX)
	MOVW CX, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX

gen_emit_copy_end:
	MOVQ BX, ret+40(FP)
	RET

// func emitCopyNoRepeat(dst []byte, offset int, length int) int
TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48
	XORQ BX, BX
	MOVQ dst_base+0(FP), AX
	MOVQ offset+24(FP), CX
	MOVQ length+32(FP), DX

	// emitCopy
	CMPL CX, $0x00010000
	JB   two_byte_offset_standalone_snappy

four_bytes_loop_back_standalone_snappy:
	CMPL DX, $0x40
	JBE  four_bytes_remain_standalone_snappy
	MOVB $0xff, (AX)
	MOVL CX, 1(AX)
	LEAL -64(DX), DX
	ADDQ $0x05, BX
	ADDQ $0x05, AX
	CMPL DX, $0x04
	JB   four_bytes_remain_standalone_snappy
	JMP  four_bytes_loop_back_standalone_snappy

four_bytes_remain_standalone_snappy:
	TESTL DX, DX
	JZ    gen_emit_copy_end_snappy
	XORL  SI, SI
	LEAL  -1(SI)(DX*4), DX
	MOVB  DL, (AX)
	MOVL  CX, 1(AX)
	ADDQ  $0x05, BX
	ADDQ  $0x05, AX
	JMP   gen_emit_copy_end_snappy

two_byte_offset_standalone_snappy:
	CMPL DX, $0x40
	JBE  two_byte_offset_short_standalone_snappy
	MOVB $0xee, (AX)
	MOVW CX, 1(AX)
	LEAL -60(DX), DX
	ADDQ $0x03, AX
	ADDQ $0x03, BX
	JMP  two_byte_offset_standalone_snappy

two_byte_offset_short_standalone_snappy:
	MOVL DX, SI
	SHLL $0x02, SI
	CMPL DX, $0x0c
	JAE  emit_copy_three_standalone_snappy
	CMPL CX, $0x00000800
	JAE  emit_copy_three_standalone_snappy
	LEAL -15(SI), SI
	MOVB CL, 1(AX)
	SHRL $0x08, CX
	SHLL $0x05, CX
	ORL  CX, SI
	MOVB SI, (AX)
	ADDQ $0x02, BX
	ADDQ $0x02, AX
	JMP  gen_emit_copy_end_snappy

emit_copy_three_standalone_snappy:
	LEAL -2(SI), SI
	MOVB SI, (AX)
	MOVW CX, 1(AX)
	ADDQ $0x03, BX
	ADDQ $0x03, AX

gen_emit_copy_end_snappy:
	MOVQ BX, ret+40(FP)
	RET

// func matchLen(a []byte, b []byte) int
// Requires: BMI
TEXT ·matchLen(SB), NOSPLIT, $0-56
	MOVQ a_base+0(FP), AX
	MOVQ b_base+24(FP), CX
	MOVQ a_len+8(FP), DX

	// matchLen
	XORL SI, SI

matchlen_loopback_16_standalone:
	CMPL DX, $0x10
	JB   matchlen_match8_standalone
	MOVQ (AX)(SI*1), BX
	MOVQ 8(AX)(SI*1), DI
	XORQ (CX)(SI*1), BX
	JNZ  matchlen_bsf_8_standalone
	XORQ 8(CX)(SI*1), DI
	JNZ  matchlen_bsf_16standalone
	LEAL -16(DX), DX
	LEAL 16(SI), SI
	JMP  matchlen_loopback_16_standalone

matchlen_bsf_16standalone:
#ifdef GOAMD64_v3
	TZCNTQ DI, DI

#else
	BSFQ DI, DI

#endif
	SARQ $0x03, DI
	LEAL 8(SI)(DI*1), SI
	JMP  gen_match_len_end

matchlen_match8_standalone:
	CMPL DX, $0x08
	JB   matchlen_match4_standalone
	MOVQ (AX)(SI*1), BX
	XORQ (CX)(SI*1), BX
	JNZ  matchlen_bsf_8_standalone
	LEAL -8(DX), DX
	LEAL 8(SI), SI
	JMP  matchlen_match4_standalone

matchlen_bsf_8_standalone:
#ifdef GOAMD64_v3
	TZCNTQ BX, BX

#else
	BSFQ BX, BX

#endif
	SARQ $0x03, BX
	LEAL (SI)(BX*1), SI
	JMP  gen_match_len_end

matchlen_match4_standalone:
	CMPL DX, $0x04
	JB   matchlen_match2_standalone
	MOVL (AX)(SI*1), BX
	CMPL (CX)(SI*1), BX
	JNE  matchlen_match2_standalone
	LEAL -4(DX), DX
	LEAL 4(SI), SI

matchlen_match2_standalone:
	CMPL DX, $0x01
	JE   matchlen_match1_standalone
	JB   gen_match_len_end
	MOVW (AX)(SI*1), BX
	CMPW (CX)(SI*1), BX
	JNE  matchlen_match1_standalone
	LEAL 2(SI), SI
	SUBL $0x02, DX
	JZ   gen_match_len_end

matchlen_match1_standalone:
	MOVB (AX)(SI*1), BL
	CMPB (CX)(SI*1), BL
	JNE  gen_match_len_end
	LEAL 1(SI), SI

gen_match_len_end:
	MOVQ SI, ret+48(FP)
	RET

// func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
// Requires: SSE2
TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64
	XORQ SI, SI
	MOVQ dst_base+0(FP), AX
	MOVQ dst_len+8(FP), CX
	MOVQ src_base+24(FP), DX
	MOVQ src_len+32(FP), BX
	LEAQ (DX)(BX*1), BX
	LEAQ -8(AX)(CX*1), CX
	XORQ DI, DI

lz4_s2_loop:
	CMPQ    DX, BX
	JAE     lz4_s2_corrupt
	CMPQ    AX, CX
	JAE     lz4_s2_dstfull
	MOVBQZX (DX), R8
	MOVQ    R8, R9
	MOVQ    R8, R10
	SHRQ    $0x04, R9
	ANDQ    $0x0f, R10
	CMPQ    R8, $0xf0
	JB      lz4_s2_ll_end

lz4_s2_ll_loop:
	INCQ    DX
	CMPQ    DX, BX
	JAE     lz4_s2_corrupt
	MOVBQZX (DX), R8
	ADDQ    R8, R9
	CMPQ    R8, $0xff
	JEQ     lz4_s2_ll_loop

lz4_s2_ll_end:
	LEAQ  (DX)(R9*1), R8
	ADDQ  $0x04, R10
	CMPQ  R8, BX
	JAE   lz4_s2_corrupt
	INCQ  DX
	INCQ  R8
	TESTQ R9, R9
	JZ    lz4_s2_lits_done
	LEAQ  (AX)(R9*1), R11
	CMPQ  R11, CX
	JAE   lz4_s2_dstfull
	ADDQ  R9, SI
	LEAL  -1(R9), R11
	CMPL  R11, $0x3c
	JB    one_byte_lz4_s2
	CMPL  R11, $0x00000100
	JB    two_bytes_lz4_s2
	CMPL  R11, $0x00010000
	JB    three_bytes_lz4_s2
	CMPL  R11, $0x01000000
	JB    four_bytes_lz4_s2
	MOVB  $0xfc, (AX)
	MOVL  R11, 1(AX)
	ADDQ  $0x05, AX
	JMP   memmove_long_lz4_s2

four_bytes_lz4_s2:
	MOVL R11, R12
	SHRL $0x10, R12
	MOVB $0xf8, (AX)
	MOVW R11, 1(AX)
	MOVB R12, 3(AX)
	ADDQ $0x04, AX
	JMP  memmove_long_lz4_s2

three_bytes_lz4_s2:
	MOVB $0xf4, (AX)
	MOVW R11, 1(AX)
	ADDQ $0x03, AX
	JMP  memmove_long_lz4_s2

two_bytes_lz4_s2:
	MOVB $0xf0, (AX)
	MOVB R11, 1(AX)
	ADDQ $0x02, AX
	CMPL R11, $0x40
	JB   memmove_lz4_s2
	JMP  memmove_long_lz4_s2

one_byte_lz4_s2:
	SHLB $0x02, R11
	MOVB R11, (AX)
	ADDQ $0x01, AX

memmove_lz4_s2:
	LEAQ (AX)(R9*1), R11

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_lz4_s2_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_lz4_s2_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_lz4_s2_memmove_move_17through32
	JMP  emit_lit_memmove_lz4_s2_memmove_move_33through64

emit_lit_memmove_lz4_s2_memmove_move_8:
	MOVQ (DX), R12
	MOVQ R12, (AX)
	JMP  memmove_end_copy_lz4_s2

emit_lit_memmove_lz4_s2_memmove_move_8through16:
	MOVQ (DX), R12
	MOVQ -8(DX)(R9*1), DX
	MOVQ R12, (AX)
	MOVQ DX, -8(AX)(R9*1)
	JMP  memmove_end_copy_lz4_s2

emit_lit_memmove_lz4_s2_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R9*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R9*1)
	JMP   memmove_end_copy_lz4_s2

emit_lit_memmove_lz4_s2_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R9*1), X2
	MOVOU -16(DX)(R9*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R9*1)
	MOVOU X3, -16(AX)(R9*1)

memmove_end_copy_lz4_s2:
	MOVQ R11, AX
	JMP  lz4_s2_lits_emit_done

memmove_long_lz4_s2:
	LEAQ (AX)(R9*1), R11

	// genMemMoveLong
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R9*1), X2
	MOVOU -16(DX)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  AX, R12
	ANDL  $0x0000001f, R12
	MOVQ  $0x00000040, R14
	SUBQ  R12, R14
	DECQ  R13
	JA    emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32
	LEAQ  -32(DX)(R14*1), R12
	LEAQ  -32(AX)(R14*1), R15

emit_lit_memmove_long_lz4_s2large_big_loop_back:
	MOVOU (R12), X4
	MOVOU 16(R12), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R12
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_lz4_s2large_big_loop_back

emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32:
	MOVOU -32(DX)(R14*1), X4
	MOVOU -16(DX)(R14*1), X5
	MOVOA X4, -32(AX)(R14*1)
	MOVOA X5, -16(AX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_lz4_s2large_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R9*1)
	MOVOU X3, -16(AX)(R9*1)
	MOVQ  R11, AX

lz4_s2_lits_emit_done:
	MOVQ R8, DX

lz4_s2_lits_done:
	CMPQ DX, BX
	JNE  lz4_s2_match
	CMPQ R10, $0x04
	JEQ  lz4_s2_done
	JMP  lz4_s2_corrupt

lz4_s2_match:
	LEAQ    2(DX), R8
	CMPQ    R8, BX
	JAE     lz4_s2_corrupt
	MOVWQZX (DX), R9
	MOVQ    R8, DX
	TESTQ   R9, R9
	JZ      lz4_s2_corrupt
	CMPQ    R9, SI
	JA      lz4_s2_corrupt
	CMPQ    R10, $0x13
	JNE     lz4_s2_ml_done

lz4_s2_ml_loop:
	MOVBQZX (DX), R8
	INCQ    DX
	ADDQ    R8, R10
	CMPQ    DX, BX
	JAE     lz4_s2_corrupt
	CMPQ    R8, $0xff
	JEQ     lz4_s2_ml_loop

lz4_s2_ml_done:
	ADDQ R10, SI
	CMPQ R9, DI
	JNE  lz4_s2_docopy

	// emitRepeat
emit_repeat_again_lz4_s2:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2

cant_repeat_two_offset_lz4_s2:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2

repeat_five_lz4_s2:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4_s2_loop

repeat_four_lz4_s2:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4_s2_loop

repeat_three_lz4_s2:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4_s2_loop

repeat_two_lz4_s2:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

repeat_two_offset_lz4_s2:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

lz4_s2_docopy:
	MOVQ R9, DI

	// emitCopy
	CMPL R10, $0x40
	JBE  two_byte_offset_short_lz4_s2
	CMPL R9, $0x00000800
	JAE  long_offset_short_lz4_s2
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB R9, 1(AX)
	MOVL R9, R11
	SHRL $0x08, R11
	SHLL $0x05, R11
	ORL  R11, R8
	MOVB R8, (AX)
	ADDQ $0x02, AX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_lz4_s2_emit_copy_short_2b

emit_repeat_again_lz4_s2_emit_copy_short_2b:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2_emit_copy_short_2b

cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2_emit_copy_short_2b
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2_emit_copy_short_2b
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2_emit_copy_short_2b
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2_emit_copy_short_2b

repeat_five_lz4_s2_emit_copy_short_2b:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4_s2_loop

repeat_four_lz4_s2_emit_copy_short_2b:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4_s2_loop

repeat_three_lz4_s2_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4_s2_loop

repeat_two_lz4_s2_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

repeat_two_offset_lz4_s2_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

long_offset_short_lz4_s2:
	MOVB $0xee, (AX)
	MOVW R9, 1(AX)
	LEAL -60(R10), R10
	ADDQ $0x03, AX

	// emitRepeat
emit_repeat_again_lz4_s2_emit_copy_short:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2_emit_copy_short
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2_emit_copy_short

cant_repeat_two_offset_lz4_s2_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2_emit_copy_short
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2_emit_copy_short
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2_emit_copy_short
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2_emit_copy_short

repeat_five_lz4_s2_emit_copy_short:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4_s2_loop

repeat_four_lz4_s2_emit_copy_short:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4_s2_loop

repeat_three_lz4_s2_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4_s2_loop

repeat_two_lz4_s2_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

repeat_two_offset_lz4_s2_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

two_byte_offset_short_lz4_s2:
	MOVL R10, R8
	SHLL $0x02, R8
	CMPL R10, $0x0c
	JAE  emit_copy_three_lz4_s2
	CMPL R9, $0x00000800
	JAE  emit_copy_three_lz4_s2
	LEAL -15(R8), R8
	MOVB R9, 1(AX)
	SHRL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R8
	MOVB R8, (AX)
	ADDQ $0x02, AX
	JMP  lz4_s2_loop

emit_copy_three_lz4_s2:
	LEAL -2(R8), R8
	MOVB R8, (AX)
	MOVW R9, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_s2_loop

lz4_s2_done:
	MOVQ dst_base+0(FP), CX
	SUBQ CX, AX
	MOVQ SI, uncompressed+48(FP)
	MOVQ AX, dstUsed+56(FP)
	RET

lz4_s2_corrupt:
	XORQ AX, AX
	LEAQ -1(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

lz4_s2_dstfull:
	XORQ AX, AX
	LEAQ -2(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

// func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
// Requires: SSE2
TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64
	XORQ SI, SI
	MOVQ dst_base+0(FP), AX
	MOVQ dst_len+8(FP), CX
	MOVQ src_base+24(FP), DX
	MOVQ src_len+32(FP), BX
	LEAQ (DX)(BX*1), BX
	LEAQ -8(AX)(CX*1), CX
	XORQ DI, DI

lz4s_s2_loop:
	CMPQ    DX, BX
	JAE     lz4s_s2_corrupt
	CMPQ    AX, CX
	JAE     lz4s_s2_dstfull
	MOVBQZX (DX), R8
	MOVQ    R8, R9
	MOVQ    R8, R10
	SHRQ    $0x04, R9
	ANDQ    $0x0f, R10
	CMPQ    R8, $0xf0
	JB      lz4s_s2_ll_end

lz4s_s2_ll_loop:
	INCQ    DX
	CMPQ    DX, BX
	JAE     lz4s_s2_corrupt
	MOVBQZX (DX), R8
	ADDQ    R8, R9
	CMPQ    R8, $0xff
	JEQ     lz4s_s2_ll_loop

lz4s_s2_ll_end:
	LEAQ  (DX)(R9*1), R8
	ADDQ  $0x03, R10
	CMPQ  R8, BX
	JAE   lz4s_s2_corrupt
	INCQ  DX
	INCQ  R8
	TESTQ R9, R9
	JZ    lz4s_s2_lits_done
	LEAQ  (AX)(R9*1), R11
	CMPQ  R11, CX
	JAE   lz4s_s2_dstfull
	ADDQ  R9, SI
	LEAL  -1(R9), R11
	CMPL  R11, $0x3c
	JB    one_byte_lz4s_s2
	CMPL  R11, $0x00000100
	JB    two_bytes_lz4s_s2
	CMPL  R11, $0x00010000
	JB    three_bytes_lz4s_s2
	CMPL  R11, $0x01000000
	JB    four_bytes_lz4s_s2
	MOVB  $0xfc, (AX)
	MOVL  R11, 1(AX)
	ADDQ  $0x05, AX
	JMP   memmove_long_lz4s_s2

four_bytes_lz4s_s2:
	MOVL R11, R12
	SHRL $0x10, R12
	MOVB $0xf8, (AX)
	MOVW R11, 1(AX)
	MOVB R12, 3(AX)
	ADDQ $0x04, AX
	JMP  memmove_long_lz4s_s2

three_bytes_lz4s_s2:
	MOVB $0xf4, (AX)
	MOVW R11, 1(AX)
	ADDQ $0x03, AX
	JMP  memmove_long_lz4s_s2

two_bytes_lz4s_s2:
	MOVB $0xf0, (AX)
	MOVB R11, 1(AX)
	ADDQ $0x02, AX
	CMPL R11, $0x40
	JB   memmove_lz4s_s2
	JMP  memmove_long_lz4s_s2

one_byte_lz4s_s2:
	SHLB $0x02, R11
	MOVB R11, (AX)
	ADDQ $0x01, AX

memmove_lz4s_s2:
	LEAQ (AX)(R9*1), R11

	// genMemMoveShort
	CMPQ R9, $0x08
	JBE  emit_lit_memmove_lz4s_s2_memmove_move_8
	CMPQ R9, $0x10
	JBE  emit_lit_memmove_lz4s_s2_memmove_move_8through16
	CMPQ R9, $0x20
	JBE  emit_lit_memmove_lz4s_s2_memmove_move_17through32
	JMP  emit_lit_memmove_lz4s_s2_memmove_move_33through64

emit_lit_memmove_lz4s_s2_memmove_move_8:
	MOVQ (DX), R12
	MOVQ R12, (AX)
	JMP  memmove_end_copy_lz4s_s2

emit_lit_memmove_lz4s_s2_memmove_move_8through16:
	MOVQ (DX), R12
	MOVQ -8(DX)(R9*1), DX
	MOVQ R12, (AX)
	MOVQ DX, -8(AX)(R9*1)
	JMP  memmove_end_copy_lz4s_s2

emit_lit_memmove_lz4s_s2_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R9*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R9*1)
	JMP   memmove_end_copy_lz4s_s2

emit_lit_memmove_lz4s_s2_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R9*1), X2
	MOVOU -16(DX)(R9*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R9*1)
	MOVOU X3, -16(AX)(R9*1)

memmove_end_copy_lz4s_s2:
	MOVQ R11, AX
	JMP  lz4s_s2_lits_emit_done

memmove_long_lz4s_s2:
	LEAQ (AX)(R9*1), R11

	// genMemMoveLong
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R9*1), X2
	MOVOU -16(DX)(R9*1), X3
	MOVQ  R9, R13
	SHRQ  $0x05, R13
	MOVQ  AX, R12
	ANDL  $0x0000001f, R12
	MOVQ  $0x00000040, R14
	SUBQ  R12, R14
	DECQ  R13
	JA    emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32
	LEAQ  -32(DX)(R14*1), R12
	LEAQ  -32(AX)(R14*1), R15

emit_lit_memmove_long_lz4s_s2large_big_loop_back:
	MOVOU (R12), X4
	MOVOU 16(R12), X5
	MOVOA X4, (R15)
	MOVOA X5, 16(R15)
	ADDQ  $0x20, R15
	ADDQ  $0x20, R12
	ADDQ  $0x20, R14
	DECQ  R13
	JNA   emit_lit_memmove_long_lz4s_s2large_big_loop_back

emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32:
	MOVOU -32(DX)(R14*1), X4
	MOVOU -16(DX)(R14*1), X5
	MOVOA X4, -32(AX)(R14*1)
	MOVOA X5, -16(AX)(R14*1)
	ADDQ  $0x20, R14
	CMPQ  R9, R14
	JAE   emit_lit_memmove_long_lz4s_s2large_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R9*1)
	MOVOU X3, -16(AX)(R9*1)
	MOVQ  R11, AX

lz4s_s2_lits_emit_done:
	MOVQ R8, DX

lz4s_s2_lits_done:
	CMPQ DX, BX
	JNE  lz4s_s2_match
	CMPQ R10, $0x03
	JEQ  lz4s_s2_done
	JMP  lz4s_s2_corrupt

lz4s_s2_match:
	CMPQ    R10, $0x03
	JEQ     lz4s_s2_loop
	LEAQ    2(DX), R8
	CMPQ    R8, BX
	JAE     lz4s_s2_corrupt
	MOVWQZX (DX), R9
	MOVQ    R8, DX
	TESTQ   R9, R9
	JZ      lz4s_s2_corrupt
	CMPQ    R9, SI
	JA      lz4s_s2_corrupt
	CMPQ    R10, $0x12
	JNE     lz4s_s2_ml_done

lz4s_s2_ml_loop:
	MOVBQZX (DX), R8
	INCQ    DX
	ADDQ    R8, R10
	CMPQ    DX, BX
	JAE     lz4s_s2_corrupt
	CMPQ    R8, $0xff
	JEQ     lz4s_s2_ml_loop

lz4s_s2_ml_done:
	ADDQ R10, SI
	CMPQ R9, DI
	JNE  lz4s_s2_docopy

	// emitRepeat
emit_repeat_again_lz4_s2:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2

cant_repeat_two_offset_lz4_s2:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2

repeat_five_lz4_s2:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4s_s2_loop

repeat_four_lz4_s2:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4s_s2_loop

repeat_three_lz4_s2:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4s_s2_loop

repeat_two_lz4_s2:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

repeat_two_offset_lz4_s2:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

lz4s_s2_docopy:
	MOVQ R9, DI

	// emitCopy
	CMPL R10, $0x40
	JBE  two_byte_offset_short_lz4_s2
	CMPL R9, $0x00000800
	JAE  long_offset_short_lz4_s2
	MOVL $0x00000001, R8
	LEAL 16(R8), R8
	MOVB R9, 1(AX)
	MOVL R9, R11
	SHRL $0x08, R11
	SHLL $0x05, R11
	ORL  R11, R8
	MOVB R8, (AX)
	ADDQ $0x02, AX
	SUBL $0x08, R10

	// emitRepeat
	LEAL -4(R10), R10
	JMP  cant_repeat_two_offset_lz4_s2_emit_copy_short_2b

emit_repeat_again_lz4_s2_emit_copy_short_2b:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2_emit_copy_short_2b
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2_emit_copy_short_2b
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2_emit_copy_short_2b

cant_repeat_two_offset_lz4_s2_emit_copy_short_2b:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2_emit_copy_short_2b
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2_emit_copy_short_2b
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2_emit_copy_short_2b
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2_emit_copy_short_2b

repeat_five_lz4_s2_emit_copy_short_2b:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4s_s2_loop

repeat_four_lz4_s2_emit_copy_short_2b:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4s_s2_loop

repeat_three_lz4_s2_emit_copy_short_2b:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4s_s2_loop

repeat_two_lz4_s2_emit_copy_short_2b:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

repeat_two_offset_lz4_s2_emit_copy_short_2b:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

long_offset_short_lz4_s2:
	MOVB $0xee, (AX)
	MOVW R9, 1(AX)
	LEAL -60(R10), R10
	ADDQ $0x03, AX

	// emitRepeat
emit_repeat_again_lz4_s2_emit_copy_short:
	MOVL R10, R8
	LEAL -4(R10), R10
	CMPL R8, $0x08
	JBE  repeat_two_lz4_s2_emit_copy_short
	CMPL R8, $0x0c
	JAE  cant_repeat_two_offset_lz4_s2_emit_copy_short
	CMPL R9, $0x00000800
	JB   repeat_two_offset_lz4_s2_emit_copy_short

cant_repeat_two_offset_lz4_s2_emit_copy_short:
	CMPL R10, $0x00000104
	JB   repeat_three_lz4_s2_emit_copy_short
	CMPL R10, $0x00010100
	JB   repeat_four_lz4_s2_emit_copy_short
	CMPL R10, $0x0100ffff
	JB   repeat_five_lz4_s2_emit_copy_short
	LEAL -16842747(R10), R10
	MOVL $0xfffb001d, (AX)
	MOVB $0xff, 4(AX)
	ADDQ $0x05, AX
	JMP  emit_repeat_again_lz4_s2_emit_copy_short

repeat_five_lz4_s2_emit_copy_short:
	LEAL -65536(R10), R10
	MOVL R10, R9
	MOVW $0x001d, (AX)
	MOVW R10, 2(AX)
	SARL $0x10, R9
	MOVB R9, 4(AX)
	ADDQ $0x05, AX
	JMP  lz4s_s2_loop

repeat_four_lz4_s2_emit_copy_short:
	LEAL -256(R10), R10
	MOVW $0x0019, (AX)
	MOVW R10, 2(AX)
	ADDQ $0x04, AX
	JMP  lz4s_s2_loop

repeat_three_lz4_s2_emit_copy_short:
	LEAL -4(R10), R10
	MOVW $0x0015, (AX)
	MOVB R10, 2(AX)
	ADDQ $0x03, AX
	JMP  lz4s_s2_loop

repeat_two_lz4_s2_emit_copy_short:
	SHLL $0x02, R10
	ORL  $0x01, R10
	MOVW R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

repeat_two_offset_lz4_s2_emit_copy_short:
	XORQ R8, R8
	LEAL 1(R8)(R10*4), R10
	MOVB R9, 1(AX)
	SARL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R10
	MOVB R10, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

two_byte_offset_short_lz4_s2:
	MOVL R10, R8
	SHLL $0x02, R8
	CMPL R10, $0x0c
	JAE  emit_copy_three_lz4_s2
	CMPL R9, $0x00000800
	JAE  emit_copy_three_lz4_s2
	LEAL -15(R8), R8
	MOVB R9, 1(AX)
	SHRL $0x08, R9
	SHLL $0x05, R9
	ORL  R9, R8
	MOVB R8, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_s2_loop

emit_copy_three_lz4_s2:
	LEAL -2(R8), R8
	MOVB R8, (AX)
	MOVW R9, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4s_s2_loop

lz4s_s2_done:
	MOVQ dst_base+0(FP), CX
	SUBQ CX, AX
	MOVQ SI, uncompressed+48(FP)
	MOVQ AX, dstUsed+56(FP)
	RET

lz4s_s2_corrupt:
	XORQ AX, AX
	LEAQ -1(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

lz4s_s2_dstfull:
	XORQ AX, AX
	LEAQ -2(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

// func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
// Requires: SSE2
TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64
	XORQ SI, SI
	MOVQ dst_base+0(FP), AX
	MOVQ dst_len+8(FP), CX
	MOVQ src_base+24(FP), DX
	MOVQ src_len+32(FP), BX
	LEAQ (DX)(BX*1), BX
	LEAQ -8(AX)(CX*1), CX

lz4_snappy_loop:
	CMPQ    DX, BX
	JAE     lz4_snappy_corrupt
	CMPQ    AX, CX
	JAE     lz4_snappy_dstfull
	MOVBQZX (DX), DI
	MOVQ    DI, R8
	MOVQ    DI, R9
	SHRQ    $0x04, R8
	ANDQ    $0x0f, R9
	CMPQ    DI, $0xf0
	JB      lz4_snappy_ll_end

lz4_snappy_ll_loop:
	INCQ    DX
	CMPQ    DX, BX
	JAE     lz4_snappy_corrupt
	MOVBQZX (DX), DI
	ADDQ    DI, R8
	CMPQ    DI, $0xff
	JEQ     lz4_snappy_ll_loop

lz4_snappy_ll_end:
	LEAQ  (DX)(R8*1), DI
	ADDQ  $0x04, R9
	CMPQ  DI, BX
	JAE   lz4_snappy_corrupt
	INCQ  DX
	INCQ  DI
	TESTQ R8, R8
	JZ    lz4_snappy_lits_done
	LEAQ  (AX)(R8*1), R10
	CMPQ  R10, CX
	JAE   lz4_snappy_dstfull
	ADDQ  R8, SI
	LEAL  -1(R8), R10
	CMPL  R10, $0x3c
	JB    one_byte_lz4_snappy
	CMPL  R10, $0x00000100
	JB    two_bytes_lz4_snappy
	CMPL  R10, $0x00010000
	JB    three_bytes_lz4_snappy
	CMPL  R10, $0x01000000
	JB    four_bytes_lz4_snappy
	MOVB  $0xfc, (AX)
	MOVL  R10, 1(AX)
	ADDQ  $0x05, AX
	JMP   memmove_long_lz4_snappy

four_bytes_lz4_snappy:
	MOVL R10, R11
	SHRL $0x10, R11
	MOVB $0xf8, (AX)
	MOVW R10, 1(AX)
	MOVB R11, 3(AX)
	ADDQ $0x04, AX
	JMP  memmove_long_lz4_snappy

three_bytes_lz4_snappy:
	MOVB $0xf4, (AX)
	MOVW R10, 1(AX)
	ADDQ $0x03, AX
	JMP  memmove_long_lz4_snappy

two_bytes_lz4_snappy:
	MOVB $0xf0, (AX)
	MOVB R10, 1(AX)
	ADDQ $0x02, AX
	CMPL R10, $0x40
	JB   memmove_lz4_snappy
	JMP  memmove_long_lz4_snappy

one_byte_lz4_snappy:
	SHLB $0x02, R10
	MOVB R10, (AX)
	ADDQ $0x01, AX

memmove_lz4_snappy:
	LEAQ (AX)(R8*1), R10

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_lz4_snappy_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_lz4_snappy_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_lz4_snappy_memmove_move_17through32
	JMP  emit_lit_memmove_lz4_snappy_memmove_move_33through64

emit_lit_memmove_lz4_snappy_memmove_move_8:
	MOVQ (DX), R11
	MOVQ R11, (AX)
	JMP  memmove_end_copy_lz4_snappy

emit_lit_memmove_lz4_snappy_memmove_move_8through16:
	MOVQ (DX), R11
	MOVQ -8(DX)(R8*1), DX
	MOVQ R11, (AX)
	MOVQ DX, -8(AX)(R8*1)
	JMP  memmove_end_copy_lz4_snappy

emit_lit_memmove_lz4_snappy_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R8*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R8*1)
	JMP   memmove_end_copy_lz4_snappy

emit_lit_memmove_lz4_snappy_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R8*1), X2
	MOVOU -16(DX)(R8*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R8*1)
	MOVOU X3, -16(AX)(R8*1)

memmove_end_copy_lz4_snappy:
	MOVQ R10, AX
	JMP  lz4_snappy_lits_emit_done

memmove_long_lz4_snappy:
	LEAQ (AX)(R8*1), R10

	// genMemMoveLong
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R8*1), X2
	MOVOU -16(DX)(R8*1), X3
	MOVQ  R8, R12
	SHRQ  $0x05, R12
	MOVQ  AX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32
	LEAQ  -32(DX)(R13*1), R11
	LEAQ  -32(AX)(R13*1), R14

emit_lit_memmove_long_lz4_snappylarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_lz4_snappylarge_big_loop_back

emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32:
	MOVOU -32(DX)(R13*1), X4
	MOVOU -16(DX)(R13*1), X5
	MOVOA X4, -32(AX)(R13*1)
	MOVOA X5, -16(AX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_lz4_snappylarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R8*1)
	MOVOU X3, -16(AX)(R8*1)
	MOVQ  R10, AX

lz4_snappy_lits_emit_done:
	MOVQ DI, DX

lz4_snappy_lits_done:
	CMPQ DX, BX
	JNE  lz4_snappy_match
	CMPQ R9, $0x04
	JEQ  lz4_snappy_done
	JMP  lz4_snappy_corrupt

lz4_snappy_match:
	LEAQ    2(DX), DI
	CMPQ    DI, BX
	JAE     lz4_snappy_corrupt
	MOVWQZX (DX), R8
	MOVQ    DI, DX
	TESTQ   R8, R8
	JZ      lz4_snappy_corrupt
	CMPQ    R8, SI
	JA      lz4_snappy_corrupt
	CMPQ    R9, $0x13
	JNE     lz4_snappy_ml_done

lz4_snappy_ml_loop:
	MOVBQZX (DX), DI
	INCQ    DX
	ADDQ    DI, R9
	CMPQ    DX, BX
	JAE     lz4_snappy_corrupt
	CMPQ    DI, $0xff
	JEQ     lz4_snappy_ml_loop

lz4_snappy_ml_done:
	ADDQ R9, SI

	// emitCopy
two_byte_offset_lz4_s2:
	CMPL R9, $0x40
	JBE  two_byte_offset_short_lz4_s2
	MOVB $0xee, (AX)
	MOVW R8, 1(AX)
	LEAL -60(R9), R9
	ADDQ $0x03, AX
	CMPQ AX, CX
	JAE  lz4_snappy_loop
	JMP  two_byte_offset_lz4_s2

two_byte_offset_short_lz4_s2:
	MOVL R9, DI
	SHLL $0x02, DI
	CMPL R9, $0x0c
	JAE  emit_copy_three_lz4_s2
	CMPL R8, $0x00000800
	JAE  emit_copy_three_lz4_s2
	LEAL -15(DI), DI
	MOVB R8, 1(AX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, DI
	MOVB DI, (AX)
	ADDQ $0x02, AX
	JMP  lz4_snappy_loop

emit_copy_three_lz4_s2:
	LEAL -2(DI), DI
	MOVB DI, (AX)
	MOVW R8, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4_snappy_loop

lz4_snappy_done:
	MOVQ dst_base+0(FP), CX
	SUBQ CX, AX
	MOVQ SI, uncompressed+48(FP)
	MOVQ AX, dstUsed+56(FP)
	RET

lz4_snappy_corrupt:
	XORQ AX, AX
	LEAQ -1(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

lz4_snappy_dstfull:
	XORQ AX, AX
	LEAQ -2(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

// func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int)
// Requires: SSE2
TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64
	XORQ SI, SI
	MOVQ dst_base+0(FP), AX
	MOVQ dst_len+8(FP), CX
	MOVQ src_base+24(FP), DX
	MOVQ src_len+32(FP), BX
	LEAQ (DX)(BX*1), BX
	LEAQ -8(AX)(CX*1), CX

lz4s_snappy_loop:
	CMPQ    DX, BX
	JAE     lz4s_snappy_corrupt
	CMPQ    AX, CX
	JAE     lz4s_snappy_dstfull
	MOVBQZX (DX), DI
	MOVQ    DI, R8
	MOVQ    DI, R9
	SHRQ    $0x04, R8
	ANDQ    $0x0f, R9
	CMPQ    DI, $0xf0
	JB      lz4s_snappy_ll_end

lz4s_snappy_ll_loop:
	INCQ    DX
	CMPQ    DX, BX
	JAE     lz4s_snappy_corrupt
	MOVBQZX (DX), DI
	ADDQ    DI, R8
	CMPQ    DI, $0xff
	JEQ     lz4s_snappy_ll_loop

lz4s_snappy_ll_end:
	LEAQ  (DX)(R8*1), DI
	ADDQ  $0x03, R9
	CMPQ  DI, BX
	JAE   lz4s_snappy_corrupt
	INCQ  DX
	INCQ  DI
	TESTQ R8, R8
	JZ    lz4s_snappy_lits_done
	LEAQ  (AX)(R8*1), R10
	CMPQ  R10, CX
	JAE   lz4s_snappy_dstfull
	ADDQ  R8, SI
	LEAL  -1(R8), R10
	CMPL  R10, $0x3c
	JB    one_byte_lz4s_snappy
	CMPL  R10, $0x00000100
	JB    two_bytes_lz4s_snappy
	CMPL  R10, $0x00010000
	JB    three_bytes_lz4s_snappy
	CMPL  R10, $0x01000000
	JB    four_bytes_lz4s_snappy
	MOVB  $0xfc, (AX)
	MOVL  R10, 1(AX)
	ADDQ  $0x05, AX
	JMP   memmove_long_lz4s_snappy

four_bytes_lz4s_snappy:
	MOVL R10, R11
	SHRL $0x10, R11
	MOVB $0xf8, (AX)
	MOVW R10, 1(AX)
	MOVB R11, 3(AX)
	ADDQ $0x04, AX
	JMP  memmove_long_lz4s_snappy

three_bytes_lz4s_snappy:
	MOVB $0xf4, (AX)
	MOVW R10, 1(AX)
	ADDQ $0x03, AX
	JMP  memmove_long_lz4s_snappy

two_bytes_lz4s_snappy:
	MOVB $0xf0, (AX)
	MOVB R10, 1(AX)
	ADDQ $0x02, AX
	CMPL R10, $0x40
	JB   memmove_lz4s_snappy
	JMP  memmove_long_lz4s_snappy

one_byte_lz4s_snappy:
	SHLB $0x02, R10
	MOVB R10, (AX)
	ADDQ $0x01, AX

memmove_lz4s_snappy:
	LEAQ (AX)(R8*1), R10

	// genMemMoveShort
	CMPQ R8, $0x08
	JBE  emit_lit_memmove_lz4s_snappy_memmove_move_8
	CMPQ R8, $0x10
	JBE  emit_lit_memmove_lz4s_snappy_memmove_move_8through16
	CMPQ R8, $0x20
	JBE  emit_lit_memmove_lz4s_snappy_memmove_move_17through32
	JMP  emit_lit_memmove_lz4s_snappy_memmove_move_33through64

emit_lit_memmove_lz4s_snappy_memmove_move_8:
	MOVQ (DX), R11
	MOVQ R11, (AX)
	JMP  memmove_end_copy_lz4s_snappy

emit_lit_memmove_lz4s_snappy_memmove_move_8through16:
	MOVQ (DX), R11
	MOVQ -8(DX)(R8*1), DX
	MOVQ R11, (AX)
	MOVQ DX, -8(AX)(R8*1)
	JMP  memmove_end_copy_lz4s_snappy

emit_lit_memmove_lz4s_snappy_memmove_move_17through32:
	MOVOU (DX), X0
	MOVOU -16(DX)(R8*1), X1
	MOVOU X0, (AX)
	MOVOU X1, -16(AX)(R8*1)
	JMP   memmove_end_copy_lz4s_snappy

emit_lit_memmove_lz4s_snappy_memmove_move_33through64:
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R8*1), X2
	MOVOU -16(DX)(R8*1), X3
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R8*1)
	MOVOU X3, -16(AX)(R8*1)

memmove_end_copy_lz4s_snappy:
	MOVQ R10, AX
	JMP  lz4s_snappy_lits_emit_done

memmove_long_lz4s_snappy:
	LEAQ (AX)(R8*1), R10

	// genMemMoveLong
	MOVOU (DX), X0
	MOVOU 16(DX), X1
	MOVOU -32(DX)(R8*1), X2
	MOVOU -16(DX)(R8*1), X3
	MOVQ  R8, R12
	SHRQ  $0x05, R12
	MOVQ  AX, R11
	ANDL  $0x0000001f, R11
	MOVQ  $0x00000040, R13
	SUBQ  R11, R13
	DECQ  R12
	JA    emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32
	LEAQ  -32(DX)(R13*1), R11
	LEAQ  -32(AX)(R13*1), R14

emit_lit_memmove_long_lz4s_snappylarge_big_loop_back:
	MOVOU (R11), X4
	MOVOU 16(R11), X5
	MOVOA X4, (R14)
	MOVOA X5, 16(R14)
	ADDQ  $0x20, R14
	ADDQ  $0x20, R11
	ADDQ  $0x20, R13
	DECQ  R12
	JNA   emit_lit_memmove_long_lz4s_snappylarge_big_loop_back

emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32:
	MOVOU -32(DX)(R13*1), X4
	MOVOU -16(DX)(R13*1), X5
	MOVOA X4, -32(AX)(R13*1)
	MOVOA X5, -16(AX)(R13*1)
	ADDQ  $0x20, R13
	CMPQ  R8, R13
	JAE   emit_lit_memmove_long_lz4s_snappylarge_forward_sse_loop_32
	MOVOU X0, (AX)
	MOVOU X1, 16(AX)
	MOVOU X2, -32(AX)(R8*1)
	MOVOU X3, -16(AX)(R8*1)
	MOVQ  R10, AX

lz4s_snappy_lits_emit_done:
	MOVQ DI, DX

lz4s_snappy_lits_done:
	CMPQ DX, BX
	JNE  lz4s_snappy_match
	CMPQ R9, $0x03
	JEQ  lz4s_snappy_done
	JMP  lz4s_snappy_corrupt

lz4s_snappy_match:
	CMPQ    R9, $0x03
	JEQ     lz4s_snappy_loop
	LEAQ    2(DX), DI
	CMPQ    DI, BX
	JAE     lz4s_snappy_corrupt
	MOVWQZX (DX), R8
	MOVQ    DI, DX
	TESTQ   R8, R8
	JZ      lz4s_snappy_corrupt
	CMPQ    R8, SI
	JA      lz4s_snappy_corrupt
	CMPQ    R9, $0x12
	JNE     lz4s_snappy_ml_done

lz4s_snappy_ml_loop:
	MOVBQZX (DX), DI
	INCQ    DX
	ADDQ    DI, R9
	CMPQ    DX, BX
	JAE     lz4s_snappy_corrupt
	CMPQ    DI, $0xff
	JEQ     lz4s_snappy_ml_loop

lz4s_snappy_ml_done:
	ADDQ R9, SI

	// emitCopy
two_byte_offset_lz4_s2:
	CMPL R9, $0x40
	JBE  two_byte_offset_short_lz4_s2
	MOVB $0xee, (AX)
	MOVW R8, 1(AX)
	LEAL -60(R9), R9
	ADDQ $0x03, AX
	CMPQ AX, CX
	JAE  lz4s_snappy_loop
	JMP  two_byte_offset_lz4_s2

two_byte_offset_short_lz4_s2:
	MOVL R9, DI
	SHLL $0x02, DI
	CMPL R9, $0x0c
	JAE  emit_copy_three_lz4_s2
	CMPL R8, $0x00000800
	JAE  emit_copy_three_lz4_s2
	LEAL -15(DI), DI
	MOVB R8, 1(AX)
	SHRL $0x08, R8
	SHLL $0x05, R8
	ORL  R8, DI
	MOVB DI, (AX)
	ADDQ $0x02, AX
	JMP  lz4s_snappy_loop

emit_copy_three_lz4_s2:
	LEAL -2(DI), DI
	MOVB DI, (AX)
	MOVW R8, 1(AX)
	ADDQ $0x03, AX
	JMP  lz4s_snappy_loop

lz4s_snappy_done:
	MOVQ dst_base+0(FP), CX
	SUBQ CX, AX
	MOVQ SI, uncompressed+48(FP)
	MOVQ AX, dstUsed+56(FP)
	RET

lz4s_snappy_corrupt:
	XORQ AX, AX
	LEAQ -1(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET

lz4s_snappy_dstfull:
	XORQ AX, AX
	LEAQ -2(AX), SI
	MOVQ SI, uncompressed+48(FP)
	RET