How to divide two unsigned long 64 bit values in x86 assembly and then returning the quotient and remainder to a C program

Question

In a separate C program, I have passed 4 parameters to an x86 ASM program.

1. dividend
2. divisor
3. Quotient pointer
4. Remainder pointer

dividend = 0xA
divisor = 0x3

Which is 10/3.

The quotient should be 3 and the remainder should be 1.

However, my quotient is returning c2 and my remainder is returning 7ffff396f687. Both of which are extremely far off of what i should be getting. I've tried debugging my ASM code and I can't figure out what the problem is.

This is what I have so far. I'm a beginner at this.

global divide64u
divide64u:
push rbp
mov rbp, rsp
mov rdx, rdi
mov rax, rsi
xor rdx, rdx 
div r10
divide64uDone:
pop rbp
ret

Answer 1

For x86_64, the args are all passed in registers, per the ABI.

So, no need to push/pop rbp/rsp

You can actually code this in C and the compiler optimizer will generate the most efficient code:

typedef unsigned long long u64;

void
davdiv(u64 div,u64 dvr,u64 *quot,u64 *rem)
{

    __asm__ (
        "\tdiv %[dvr]\n"
    :   [quot] "=a" (*quot),
        [rem] "=d" (*rem)
    :   [div] "a" (div),
        [dvr] "r" (dvr),
        "d" (0));
}

void
mydiv(u64 div,u64 dvr,u64 *quot,u64 *rem)
{

    __asm__ __volatile__(
        "\txor  %%edx,%%edx\n"
        "\tmov  %[div],%%rax\n"
        "\tdiv  %[dvr]\n"
        "\tmov  %%rax,%[quot]\n"
        "\tmov  %%rdx,%[rem]\n"
    :   [quot] "=m" (*quot),
        [rem] "=m" (*rem)
    :   [div] "r" (div),
        [dvr] "r" (dvr)
    :   "rax", "rdx");
}

void
cpldiv(u64 div,u64 dvr,u64 *quot,u64 *rem)
{

    *quot = div / dvr;
    *rem = div % dvr;
}

u64
cplretA(u64 div,u64 dvr,u64 *rem)
{
    u64 quot;

    quot = div / dvr;
    *rem = div % dvr;

    return quot;
}

u64
cplretB(u64 div,u64 dvr,u64 *quot)
{
    u64 rem;

    *quot = div / dvr;
    rem = div % dvr;

    return rem;
}

Here is the disassembly of the above compiled with -O2:

div2.o:     file format elf64-x86-64

Disassembly of section .text:

0000000000000000 <davdiv>:
   0:   49 89 d0                mov    %rdx,%r8
   3:   48 89 f8                mov    %rdi,%rax
   6:   31 d2                   xor    %edx,%edx
   8:   48 f7 f6                div    %rsi
   b:   49 89 00                mov    %rax,(%r8)
   e:   48 89 11                mov    %rdx,(%rcx)
  11:   c3                      retq
  12:   66 66 2e 0f 1f 84 00    data16 nopw %cs:0x0(%rax,%rax,1)
  19:   00 00 00 00
  1d:   0f 1f 00                nopl   (%rax)

0000000000000020 <mydiv>:
  20:   49 89 d0                mov    %rdx,%r8
  23:   31 d2                   xor    %edx,%edx
  25:   48 89 f8                mov    %rdi,%rax
  28:   48 f7 f6                div    %rsi
  2b:   49 89 00                mov    %rax,(%r8)
  2e:   48 89 11                mov    %rdx,(%rcx)
  31:   c3                      retq
  32:   66 66 2e 0f 1f 84 00    data16 nopw %cs:0x0(%rax,%rax,1)
  39:   00 00 00 00
  3d:   0f 1f 00                nopl   (%rax)

0000000000000040 <cpldiv>:
  40:   48 89 f8                mov    %rdi,%rax
  43:   48 89 d7                mov    %rdx,%rdi
  46:   31 d2                   xor    %edx,%edx
  48:   48 f7 f6                div    %rsi
  4b:   48 89 07                mov    %rax,(%rdi)
  4e:   48 89 11                mov    %rdx,(%rcx)
  51:   c3                      retq
  52:   66 66 2e 0f 1f 84 00    data16 nopw %cs:0x0(%rax,%rax,1)
  59:   00 00 00 00
  5d:   0f 1f 00                nopl   (%rax)

0000000000000060 <cplretA>:
  60:   48 89 d1                mov    %rdx,%rcx
  63:   48 89 f8                mov    %rdi,%rax
  66:   31 d2                   xor    %edx,%edx
  68:   48 f7 f6                div    %rsi
  6b:   48 89 11                mov    %rdx,(%rcx)
  6e:   c3                      retq
  6f:   90                      nop

0000000000000070 <cplretB>:
  70:   48 89 d1                mov    %rdx,%rcx
  73:   48 89 f8                mov    %rdi,%rax
  76:   31 d2                   xor    %edx,%edx
  78:   48 f7 f6                div    %rsi
  7b:   48 89 01                mov    %rax,(%rcx)
  7e:   48 89 d0                mov    %rdx,%rax
  81:   c3                      retq