N-Queens puzzle back tracing solution, resulting more outputs than expected

Question

I recently tried to solve the N-queens problem, My code worked for board size till 5, starting from 6 it gives me wrong result. What would be the issue in this code ?

I tried to investigate and debug my code many times, but couldn't figure out what is the issue.

input-output-Expected
1   -   1   -   1
2   -   0   -   0
3   -   0   -   0
4   -   2   -   2
5   -   10  -   10
6   -   19  -   4
7   -   98  -   40

class Solution {
    int count = 0;
    int size = 0;

    public int totalNQueens(int n) {
        this.size = n;
        
        countValidSolutions(0, 1, new HashSet<>(), new HashSet<>());;
        
        return this.count;
    }
    
    public void countValidSolutions(int row, int queen, Set<Integer> filled, Set<Pair> pairs) {
        
        if (row >= this.size || queen > this.size) return;
        
        // try all columns in this row for this queen
        for (int col = 0; col < this.size; col++) {
            if (!isValidCell(row, col, filled, pairs))
                continue;
            
            if (row == this.size - 1) {
                this.count++;
                continue;
            }
            
            filled.add(col);
            inValidateDiagonal(row, col, pairs);
            countValidSolutions(row+1, queen+1, filled, pairs);
            filled.remove(col);
            ValidateDiagonal(row, col, pairs);
        }
        
    }
    
    public void ValidateDiagonal(int row, int col, Set<Pair> pairs) {
        int down = row+1; int left = col-1;
        while (down < this.size && left > -1) {
            Pair<Integer, Integer> p1 = new Pair<>(down, left);
            pairs.remove(p1);
            down++; left--;
        }
        down = row+1; 
        int right = col+1;
        
        while (down < this.size && right < this.size) {
            Pair<Integer, Integer> p1 = new Pair<>(down, right);
            pairs.remove(p1);
            down++; right++;
        }
    }
    
    public void inValidateDiagonal(int row, int col, Set<Pair> pairs) {
        int down = row+1; int left = col-1;
        while (down < this.size && left > -1) {
            Pair<Integer, Integer> p1 = new Pair<>(down, left);
            pairs.add(p1);
            down++; left--;
        }
        down = row+1;
        int right = col+1;
        
        while (down < this.size && right < this.size) {
            Pair<Integer, Integer> p1 = new Pair<>(down, right);
            pairs.add(p1);
            down++; right++;
        }
    }
    
    public boolean isValidCell(int row, int col, Set<Integer> filled, Set<Pair> pairs) {
        Pair<Integer, Integer> p = new Pair<>(row, col);
        if ( !filled.contains(col) && !pairs.contains(p) )
            return true;
        
        return false;
    }
}

the output is way larger than it is supposed to be. Thanks in advance.