Recyclerview dont refresh after removed item from firestore

Question

What i want to do is to show to the user only person who are in same city as him for that i used location and geocoder and i retrieve the data and store it in firestore and i use recyclerview to display the list of user who are in the same area, now my problem is when a new user is added or change his location to be the same as the current user, firestore data are updating normally and the recyclerview are refreshed automatically, but if a user quit the city (change his location) firestore is updating but the recyclerview is not. Now i dont know if i can just adjust my code and using notifyItemRemoved() or i should create a new collection in firestore for each city and add users who are there. There is my main code

private void getUsers() { //Get only users who are in same city
    DocumentReference currentUser = userRef.document(FirebaseAuth.getInstance().getUid());

    userRef.addSnapshotListener(new EventListener<QuerySnapshot>() {
        @Override
        public void onEvent(@Nullable QuerySnapshot value, @Nullable FirebaseFirestoreException error) {

            if (error != null){
                Log.e(TAG, "Firestore error" + error.getMessage());
                return;
            }
            if(value != null){
                for (DocumentChange doc : value.getDocumentChanges()){
                    User user = doc.getDocument().toObject(User.class);
                    mUserList.add(user); // userlist contain all user
                    Log.d(TAG, "userlist1: "+mUserList);
                }
                for (User user1 : mUserList){
                    currentUser.get().addOnSuccessListener(new OnSuccessListener<DocumentSnapshot>() {
                        @SuppressLint("NotifyDataSetChanged")
                        @Override
                        public void onSuccess(DocumentSnapshot documentSnapshot) {
                            if (user1.getLocation() != null 
                                    && user1.getLocation().equals(documentSnapshot.toObject(User.class).getLocation())){
                                mUserList2.add(user1); // userlist2 contain only user in same city}
                            mUserAdapter.notifyDataSetChanged();
                        }
                    });
                }
            }
        }
    });
}


private void getUserPosition(){ // retrieve the user location and store it in firestore 
    fusedLocationProviderClient.getLastLocation().addOnCompleteListener(new OnCompleteListener<Location>() {
        @Override
        public void onComplete(@NonNull Task<Location> task) {
            //Initialize location
            Location location = task.getResult();
            if (location != null){
                try {
                    //Initialize GeoCoder
                    Geocoder geocoder = new Geocoder(MainActivity.this,
                            Locale.getDefault());
                    //Initialize address list
                    List<Address> addresses = geocoder.getFromLocation(
                            location.getLatitude(), location.getLongitude(), 1
                    );
                    //setLocation in firestore
                    userRef.document(FirebaseAuth.getInstance().getUid())
                            .update("location", ""+Html.fromHtml(addresses.get(0).
                            getCountryName()+", "+addresses.get(0).getLocality()));
                }catch (IOException e){
                    e.printStackTrace();
                }
            }
        }
    });
}

EDITB 1 My adapter class : private Context mContext; private ArrayList mUserList;

public UserAdapter(Context mContext, ArrayList<User> mUserList) {
    this.mContext = mContext;
    this.mUserList = mUserList;
}

@NonNull
@Override
public UserHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) {
    View v = LayoutInflater.from(mContext).inflate(R.layout.user_item, parent, false);
    return new UserHolder(v);
}

@Override
public void onBindViewHolder(@NonNull UserHolder holder, int position) {

    User mUser = mUserList.get(position);
    holder.username.setText(mUser.getUsername());
    holder.email.setText(mUser.getEmail());

}

@Override
public int getItemCount() {
    return mUserList.size();
}

public static class UserHolder extends RecyclerView.ViewHolder {

    private TextView username;
    private TextView email;

    public UserHolder(View itemView) {
        super(itemView);
        username = itemView.findViewById(R.id.username);
        email = itemView.findViewById(R.id.email);
    }
}